diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib/rules')
217 files changed, 42819 insertions, 0 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/rules/RuleConfiguredTargetFactory.java b/src/main/java/com/google/devtools/build/lib/rules/RuleConfiguredTargetFactory.java new file mode 100644 index 0000000000..45df124750 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/RuleConfiguredTargetFactory.java @@ -0,0 +1,25 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules; + +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.packages.RuleClass; + +/** + * A shortcut class to the appropriate specialization of {@code RuleClass.ConfiguredTargetFactory}. + */ +public interface RuleConfiguredTargetFactory + extends RuleClass.ConfiguredTargetFactory<ConfiguredTarget, RuleContext> { +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkAttr.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkAttr.java new file mode 100644 index 0000000000..dfd6a9276d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkAttr.java @@ -0,0 +1,350 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules; + +import static com.google.devtools.build.lib.syntax.SkylarkFunction.castList; + +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition; +import com.google.devtools.build.lib.packages.Attribute.SkylarkLateBound; +import com.google.devtools.build.lib.packages.SkylarkFileType; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.packages.Type.ConversionException; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.FuncallExpression; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param; +import com.google.devtools.build.lib.syntax.SkylarkCallbackFunction; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.syntax.SkylarkFunction; +import com.google.devtools.build.lib.syntax.SkylarkList; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.syntax.UserDefinedFunction; +import com.google.devtools.build.lib.util.FileTypeSet; + +import java.util.Map; + +/** + * A helper class to provide Attr module in Skylark. + */ +@SkylarkModule(name = "attr", namespace = true, onlyLoadingPhase = true, + doc = "Module for creating new attributes. " + + "They are only for use with the <code>rule</code> function.") +public final class SkylarkAttr { + + private static final String MANDATORY_DOC = + "set to true if users have to explicitely specify the value"; + + private static final String ALLOW_FILES_DOC = + "whether File targets are allowed. Can be True, False (default), or " + + "a FileType filter."; + + private static final String ALLOW_RULES_DOC = + "which rule targets (name of the classes) are allowed." + + "This is deprecated (kept only for compatiblity), use providers instead."; + + private static final String FLAGS_DOC = + "deprecated, will be removed"; + + private static final String DEFAULT_DOC = + "sets the default value of the attribute."; + + private static final String CONFIGURATION_DOC = + "configuration of the attribute. " + + "For example, use DATA_CFG or HOST_CFG."; + + private static final String EXECUTABLE_DOC = + "set to True if the labels have to be executable. Access the labels with " + + "ctx.executable.<attribute_name>"; + + private static Attribute.Builder<?> createAttribute(Type<?> type, Map<String, Object> arguments, + FuncallExpression ast, SkylarkEnvironment env) throws EvalException, ConversionException { + final Location loc = ast.getLocation(); + // We use an empty name now so that we can set it later. + // This trick makes sense only in the context of Skylark (builtin rules should not use it). + Attribute.Builder<?> builder = Attribute.attr("", type); + + Object defaultValue = arguments.get("default"); + if (defaultValue != null) { + if (defaultValue instanceof UserDefinedFunction) { + // Late bound attribute. Non label type attributes already caused a type check error. + builder.value(new SkylarkLateBound( + new SkylarkCallbackFunction((UserDefinedFunction) defaultValue, ast, env))); + } else { + builder.defaultValue(defaultValue); + } + } + + for (String flag : castList(arguments.get("flags"), String.class)) { + builder.setPropertyFlag(flag); + } + + if (arguments.containsKey("mandatory") && (Boolean) arguments.get("mandatory")) { + builder.setPropertyFlag("MANDATORY"); + } + + if (arguments.containsKey("executable") && (Boolean) arguments.get("executable")) { + builder.setPropertyFlag("EXECUTABLE"); + } + + if (arguments.containsKey("single_file") && (Boolean) arguments.get("single_file")) { + builder.setPropertyFlag("SINGLE_ARTIFACT"); + } + + if (arguments.containsKey("allow_files")) { + Object fileTypesObj = arguments.get("allow_files"); + if (fileTypesObj == Boolean.TRUE) { + builder.allowedFileTypes(FileTypeSet.ANY_FILE); + } else if (fileTypesObj == Boolean.FALSE) { + builder.allowedFileTypes(FileTypeSet.NO_FILE); + } else if (fileTypesObj instanceof SkylarkFileType) { + builder.allowedFileTypes(((SkylarkFileType) fileTypesObj).getFileTypeSet()); + } else { + throw new EvalException(loc, "allow_files should be a boolean or a filetype object."); + } + } else if (type.equals(Type.LABEL) || type.equals(Type.LABEL_LIST)) { + builder.allowedFileTypes(FileTypeSet.NO_FILE); + } + + Object ruleClassesObj = arguments.get("allow_rules"); + if (ruleClassesObj != null) { + builder.allowedRuleClasses(castList(ruleClassesObj, String.class, + "allowed rule classes for attribute definition")); + } + + if (arguments.containsKey("providers")) { + builder.mandatoryProviders(castList(arguments.get("providers"), String.class)); + } + + if (arguments.containsKey("cfg")) { + builder.cfg((ConfigurationTransition) arguments.get("cfg")); + } + return builder; + } + + private static Object createAttribute(Map<String, Object> kwargs, Type<?> type, + FuncallExpression ast, Environment env) throws EvalException { + try { + return createAttribute(type, kwargs, ast, (SkylarkEnvironment) env); + } catch (ConversionException e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + } + + @SkylarkBuiltin(name = "int", doc = + "Creates an attribute of type int.", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = Integer.class, + doc = DEFAULT_DOC + " If not specified, default is 0."), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction integer = new SkylarkFunction("int") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.INTEGER, ast, env); + } + }; + + @SkylarkBuiltin(name = "string", doc = + "Creates an attribute of type string.", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = String.class, + doc = DEFAULT_DOC + " If not specified, default is \"\"."), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction string = new SkylarkFunction("string") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.STRING, ast, env); + } + }; + + @SkylarkBuiltin(name = "label", doc = + "Creates an attribute of type Label. " + + "It is the only way to specify a dependency to another target. " + + "If you need a dependency that the user cannot overwrite, make the attribute " + + "private (starts with <code>_</code>).", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = Label.class, callbackEnabled = true, + doc = DEFAULT_DOC + " If not specified, default is None. " + + "Use the <code>Label</code> function to specify a default value."), + @Param(name = "executable", type = Boolean.class, doc = EXECUTABLE_DOC), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "allow_files", doc = ALLOW_FILES_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "providers", type = SkylarkList.class, generic1 = String.class, + doc = "mandatory providers every dependency has to have"), + @Param(name = "allow_rules", type = SkylarkList.class, generic1 = String.class, + doc = ALLOW_RULES_DOC), + @Param(name = "single_file", doc = + "if true, the label must correspond to a single File. " + + "Access it through ctx.file.<attribute_name>."), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction label = new SkylarkFunction("label") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.LABEL, ast, env); + } + }; + + @SkylarkBuiltin(name = "string_list", doc = + "Creates an attribute of type list of strings", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = SkylarkList.class, generic1 = String.class, + doc = DEFAULT_DOC + " If not specified, default is []."), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, + doc = CONFIGURATION_DOC)}) + private static SkylarkFunction stringList = new SkylarkFunction("string_list") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.STRING_LIST, ast, env); + } + }; + + @SkylarkBuiltin(name = "label_list", doc = + "Creates an attribute of type list of labels. " + + "See <code>label</code> for more information.", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = SkylarkList.class, generic1 = Label.class, + callbackEnabled = true, + doc = DEFAULT_DOC + " If not specified, default is []. " + + "Use the <code>Label</code> function to specify a default value."), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "allow_files", doc = ALLOW_FILES_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "allow_rules", type = SkylarkList.class, generic1 = String.class, + doc = ALLOW_RULES_DOC), + @Param(name = "providers", type = SkylarkList.class, generic1 = String.class, + doc = "mandatory providers every dependency has to have"), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction labelList = new SkylarkFunction("label_list") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.LABEL_LIST, ast, env); + } + }; + + @SkylarkBuiltin(name = "bool", doc = + "Creates an attribute of type bool. Its default value is False.", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = Boolean.class, doc = DEFAULT_DOC), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction bool = new SkylarkFunction("bool") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.BOOLEAN, ast, env); + } + }; + + @SkylarkBuiltin(name = "output", doc = + "Creates an attribute of type output. Its default value is None. " + + "The user provides a file name (string) and the rule must create an action that " + + "generates the file.", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = Label.class, doc = DEFAULT_DOC), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction output = new SkylarkFunction("output") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.OUTPUT, ast, env); + } + }; + + @SkylarkBuiltin(name = "output_list", doc = + "Creates an attribute of type list of outputs. Its default value is []. " + + "See <code>output</code> above for more information.", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = SkylarkList.class, generic1 = Label.class, doc = DEFAULT_DOC), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction outputList = new SkylarkFunction("output_list") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.OUTPUT_LIST, ast, env); + } + }; + + @SkylarkBuiltin(name = "string_dict", doc = + "Creates an attribute of type dictionary, mapping from string to string. " + + "Its default value is {}.", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = Map.class, doc = DEFAULT_DOC), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction stringDict = new SkylarkFunction("string_dict") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.STRING_DICT, ast, env); + } + }; + + @SkylarkBuiltin(name = "license", doc = + "Creates an attribute of type license. Its default value is NO_LICENSE.", + // TODO(bazel-team): Implement proper license support for Skylark. + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", doc = DEFAULT_DOC), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction license = new SkylarkFunction("license") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.LICENSE, ast, env); + } + }; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkCommandLine.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkCommandLine.java new file mode 100644 index 0000000000..e51805e3e3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkCommandLine.java @@ -0,0 +1,89 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules; + +import com.google.common.base.Function; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param; +import com.google.devtools.build.lib.syntax.SkylarkFunction.SimpleSkylarkFunction; +import com.google.devtools.build.lib.syntax.SkylarkList; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.syntax.SkylarkNestedSet; + +import java.util.Map; + +/** + * A Skylark module class to create memory efficient command lines. + */ +@SkylarkModule(name = "cmd_helper", namespace = true, + doc = "Module for creating memory efficient command lines.") +public class SkylarkCommandLine { + + @SkylarkBuiltin(name = "join_paths", + doc = "Creates a single command line argument joining the paths of a set " + + "of files on the separator string.", + objectType = SkylarkCommandLine.class, + returnType = String.class, + mandatoryParams = { + @Param(name = "separator", type = String.class, doc = "the separator string to join on"), + @Param(name = "files", type = SkylarkNestedSet.class, generic1 = Artifact.class, + doc = "the files to concatenate")}) + private static SimpleSkylarkFunction joinPaths = + new SimpleSkylarkFunction("join_paths") { + @Override + public Object call(Map<String, Object> params, Location loc) + throws EvalException { + final String separator = (String) params.get("separator"); + final NestedSet<Artifact> artifacts = + ((SkylarkNestedSet) params.get("files")).getSet(Artifact.class); + // TODO(bazel-team): lazy evaluate + return Artifact.joinExecPaths(separator, artifacts); + } + }; + + // TODO(bazel-team): this method should support sets of objects and substitute all struct fields. + @SkylarkBuiltin(name = "template", + doc = "Transforms a set of files to a list of strings using the template string.", + objectType = SkylarkCommandLine.class, + returnType = SkylarkList.class, + mandatoryParams = { + @Param(name = "items", type = SkylarkNestedSet.class, generic1 = Artifact.class, + doc = "The set of structs to transform."), + @Param(name = "template", type = String.class, + doc = "The template to use for the transformation, %{path} and %{short_path} " + + "being substituted with the corresponding fields of each file.")}) + private static SimpleSkylarkFunction template = new SimpleSkylarkFunction("template") { + @Override + public Object call(Map<String, Object> params, Location loc) + throws EvalException { + final String template = (String) params.get("template"); + SkylarkNestedSet items = (SkylarkNestedSet) params.get("items"); + return SkylarkList.lazyList(Iterables.transform(items, new Function<Object, String>() { + @Override + public String apply(Object input) { + Artifact artifact = (Artifact) input; + return template + .replace("%{path}", artifact.getExecPathString()) + .replace("%{short_path}", artifact.getRootRelativePathString()); + } + }), String.class); + } + }; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkModules.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkModules.java new file mode 100644 index 0000000000..ae81f81a12 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkModules.java @@ -0,0 +1,198 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.MethodLibrary; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.Function; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.syntax.SkylarkFunction; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.syntax.SkylarkType; +import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType; +import com.google.devtools.build.lib.syntax.ValidationEnvironment; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +/** + * A class to handle all Skylark modules, to create and setup Validation and regular Environments. + */ +public class SkylarkModules { + + public static final ImmutableList<Class<?>> MODULES = ImmutableList.of( + SkylarkAttr.class, + SkylarkCommandLine.class, + SkylarkRuleClassFunctions.class, + SkylarkRuleImplementationFunctions.class); + + private static final ImmutableMap<Class<?>, ImmutableList<Function>> FUNCTION_MAP; + private static final ImmutableMap<String, Object> OBJECTS; + + static { + try { + ImmutableMap.Builder<Class<?>, ImmutableList<Function>> functionMap = ImmutableMap.builder(); + ImmutableMap.Builder<String, Object> objects = ImmutableMap.builder(); + for (Class<?> moduleClass : MODULES) { + if (moduleClass.isAnnotationPresent(SkylarkModule.class)) { + objects.put(moduleClass.getAnnotation(SkylarkModule.class).name(), + moduleClass.newInstance()); + } + ImmutableList.Builder<Function> functions = ImmutableList.builder(); + collectSkylarkFunctionsAndObjectsFromFields(moduleClass, functions, objects); + functionMap.put(moduleClass, functions.build()); + } + FUNCTION_MAP = functionMap.build(); + OBJECTS = objects.build(); + } catch (InstantiationException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + /** + * Returns a new SkylarkEnvironment with the elements of the Skylark modules. + */ + public static SkylarkEnvironment getNewEnvironment( + EventHandler eventHandler, String astFileContentHashCode) { + SkylarkEnvironment env = new SkylarkEnvironment(eventHandler, astFileContentHashCode); + setupEnvironment(env); + return env; + } + + @VisibleForTesting + public static SkylarkEnvironment getNewEnvironment(EventHandler eventHandler) { + return getNewEnvironment(eventHandler, null); + } + + private static void setupEnvironment(Environment env) { + MethodLibrary.setupMethodEnvironment(env); + for (Map.Entry<Class<?>, ImmutableList<Function>> entry : FUNCTION_MAP.entrySet()) { + for (Function function : entry.getValue()) { + if (function.getObjectType() != null) { + env.registerFunction(function.getObjectType(), function.getName(), function); + } else { + env.update(function.getName(), function); + } + } + } + for (Map.Entry<String, Object> entry : OBJECTS.entrySet()) { + env.update(entry.getKey(), entry.getValue()); + } + } + + /** + * Returns a new ValidationEnvironment with the elements of the Skylark modules. + */ + public static ValidationEnvironment getValidationEnvironment() { + return getValidationEnvironment(ImmutableMap.<String, SkylarkType>of()); + } + + /** + * Returns a new ValidationEnvironment with the elements of the Skylark modules and extraObjects. + */ + public static ValidationEnvironment getValidationEnvironment( + ImmutableMap<String, SkylarkType> extraObjects) { + Map<SkylarkType, Map<String, SkylarkType>> builtIn = new HashMap<>(); + Map<String, SkylarkType> global = new HashMap<>(); + builtIn.put(SkylarkType.GLOBAL, global); + collectSkylarkTypesFromFields(Environment.class, builtIn); + for (Class<?> moduleClass : MODULES) { + if (moduleClass.isAnnotationPresent(SkylarkModule.class)) { + global.put(moduleClass.getAnnotation(SkylarkModule.class).name(), + SkylarkType.of(moduleClass)); + } + } + global.put("native", SkylarkType.UNKNOWN); + MethodLibrary.setupValidationEnvironment(builtIn); + for (Class<?> module : MODULES) { + collectSkylarkTypesFromFields(module, builtIn); + } + global.putAll(extraObjects); + return new ValidationEnvironment(CollectionUtils.toImmutable(builtIn)); + } + + /** + * Collects the SkylarkFunctions from the fields of the class of the object parameter + * and adds them into the builder. + */ + private static void collectSkylarkFunctionsAndObjectsFromFields(Class<?> type, + ImmutableList.Builder<Function> functions, ImmutableMap.Builder<String, Object> objects) { + try { + for (Field field : type.getDeclaredFields()) { + if (field.isAnnotationPresent(SkylarkBuiltin.class)) { + // Fields in Skylark modules are sometimes private. Nevertheless they have to + // be annotated with SkylarkBuiltin. + field.setAccessible(true); + SkylarkBuiltin annotation = field.getAnnotation(SkylarkBuiltin.class); + if (SkylarkFunction.class.isAssignableFrom(field.getType())) { + SkylarkFunction function = (SkylarkFunction) field.get(null); + if (!function.isConfigured()) { + function.configure(annotation); + } + functions.add(function); + } else { + objects.put(annotation.name(), field.get(null)); + } + } + } + } catch (IllegalArgumentException | IllegalAccessException e) { + // This should never happen. + throw new RuntimeException(e); + } + } + + /** + * Collects the SkylarkFunctions from the fields of the class of the object parameter + * and adds their class and their corresponding return value to the builder. + */ + private static void collectSkylarkTypesFromFields(Class<?> classObject, + Map<SkylarkType, Map<String, SkylarkType>> builtIn) { + for (Field field : classObject.getDeclaredFields()) { + if (field.isAnnotationPresent(SkylarkBuiltin.class)) { + SkylarkBuiltin annotation = field.getAnnotation(SkylarkBuiltin.class); + if (SkylarkFunction.class.isAssignableFrom(field.getType())) { + try { + // TODO(bazel-team): infer the correct types. + SkylarkType objectType = annotation.objectType().equals(Object.class) + ? SkylarkType.GLOBAL + : SkylarkType.of(annotation.objectType()); + if (!builtIn.containsKey(objectType)) { + builtIn.put(objectType, new HashMap<String, SkylarkType>()); + } + // TODO(bazel-team): add parameters to SkylarkFunctionType + SkylarkType returnType = SkylarkType.getReturnType(annotation); + builtIn.get(objectType).put(annotation.name(), + SkylarkFunctionType.of(annotation.name(), returnType)); + } catch (IllegalArgumentException e) { + // This should never happen. + throw new RuntimeException(e); + } + } else if (Function.class.isAssignableFrom(field.getType())) { + builtIn.get(SkylarkType.GLOBAL).put(annotation.name(), + SkylarkFunctionType.of(annotation.name(), SkylarkType.UNKNOWN)); + } else { + builtIn.get(SkylarkType.GLOBAL).put(annotation.name(), SkylarkType.of(field.getType())); + } + } + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleClassFunctions.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleClassFunctions.java new file mode 100644 index 0000000000..39b8836f64 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleClassFunctions.java @@ -0,0 +1,430 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.DATA; +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.INTEGER; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.RunUnder; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition; +import com.google.devtools.build.lib.packages.Attribute.LateBoundLabel; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SkylarkImplicitOutputsFunctionWithCallback; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SkylarkImplicitOutputsFunctionWithMap; +import com.google.devtools.build.lib.packages.Package.NameConflictException; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.PackageFactory.PackageContext; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; +import com.google.devtools.build.lib.packages.RuleFactory; +import com.google.devtools.build.lib.packages.RuleFactory.InvalidRuleException; +import com.google.devtools.build.lib.packages.SkylarkFileType; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.TestSize; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.packages.Type.ConversionException; +import com.google.devtools.build.lib.syntax.AbstractFunction; +import com.google.devtools.build.lib.syntax.ClassObject; +import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.Environment.NoSuchVariableException; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.EvalUtils; +import com.google.devtools.build.lib.syntax.FuncallExpression; +import com.google.devtools.build.lib.syntax.Function; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param; +import com.google.devtools.build.lib.syntax.SkylarkCallbackFunction; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.syntax.SkylarkFunction; +import com.google.devtools.build.lib.syntax.SkylarkFunction.SimpleSkylarkFunction; +import com.google.devtools.build.lib.syntax.SkylarkList; +import com.google.devtools.build.lib.syntax.UserDefinedFunction; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +/** + * A helper class to provide an easier API for Skylark rule definitions. + * This is experimental code. + */ +public class SkylarkRuleClassFunctions { + + //TODO(bazel-team): proper enum support + @SkylarkBuiltin(name = "DATA_CFG", returnType = ConfigurationTransition.class, + doc = "The default runfiles collection state.") + private static final Object dataTransition = ConfigurationTransition.DATA; + + @SkylarkBuiltin(name = "HOST_CFG", returnType = ConfigurationTransition.class, + doc = "The default runfiles collection state.") + private static final Object hostTransition = ConfigurationTransition.HOST; + + private static final Attribute.ComputedDefault DEPRECATION = + new Attribute.ComputedDefault() { + @Override + public Object getDefault(AttributeMap rule) { + return rule.getPackageDefaultDeprecation(); + } + }; + + private static final Attribute.ComputedDefault TEST_ONLY = + new Attribute.ComputedDefault() { + @Override + public Object getDefault(AttributeMap rule) { + return rule.getPackageDefaultTestOnly(); + } + }; + + private static final LateBoundLabel<BuildConfiguration> RUN_UNDER = + new LateBoundLabel<BuildConfiguration>() { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + RunUnder runUnder = configuration.getRunUnder(); + return runUnder == null ? null : runUnder.getLabel(); + } + }; + + // TODO(bazel-team): Copied from ConfiguredRuleClassProvider for the transition from built-in + // rules to skylark extensions. Using the same instance would require a large refactoring. + // If we don't want to support old built-in rules and Skylark simultaneously + // (except for transition phase) it's probably OK. + private static LoadingCache<String, Label> labelCache = + CacheBuilder.newBuilder().build(new CacheLoader<String, Label>() { + @Override + public Label load(String from) throws Exception { + try { + return Label.parseAbsolute(from); + } catch (Label.SyntaxException e) { + throw new Exception(from); + } + } + }); + + // TODO(bazel-team): Remove the code duplication (BaseRuleClasses and this class). + private static final RuleClass baseRule = + BaseRuleClasses.commonCoreAndSkylarkAttributes( + new RuleClass.Builder("$base_rule", RuleClassType.ABSTRACT, true)) + .add(attr("expect_failure", STRING)) + .build(); + + private static final RuleClass testBaseRule = + new RuleClass.Builder("$test_base_rule", RuleClassType.ABSTRACT, true, baseRule) + .add(attr("size", STRING).value("medium").taggable() + .nonconfigurable("used in loading phase rule validation logic")) + .add(attr("timeout", STRING).taggable() + .nonconfigurable("used in loading phase rule validation logic").value( + new Attribute.ComputedDefault() { + @Override + public Object getDefault(AttributeMap rule) { + TestSize size = TestSize.getTestSize(rule.get("size", Type.STRING)); + if (size != null) { + String timeout = size.getDefaultTimeout().toString(); + if (timeout != null) { + return timeout; + } + } + return "illegal"; + } + })) + .add(attr("flaky", BOOLEAN).value(false).taggable() + .nonconfigurable("taggable - called in Rule.getRuleTags")) + .add(attr("shard_count", INTEGER).value(-1)) + .add(attr("local", BOOLEAN).value(false).taggable() + .nonconfigurable("policy decision: this should be consistent across configurations")) + .add(attr("$test_runtime", LABEL_LIST).cfg(HOST).value(ImmutableList.of( + labelCache.getUnchecked("//tools/test:runtime")))) + .add(attr(":run_under", LABEL).cfg(DATA).value(RUN_UNDER)) + .build(); + + /** + * In native code, private values start with $. + * In Skylark, private values start with _, because of the grammar. + */ + private static String attributeToNative(String oldName, Location loc, boolean isLateBound) + throws EvalException { + if (oldName.isEmpty()) { + throw new EvalException(loc, "Attribute name cannot be empty"); + } + if (isLateBound) { + if (oldName.charAt(0) != '_') { + throw new EvalException(loc, "When an attribute value is a function, " + + "the attribute must be private (start with '_')"); + } + return ":" + oldName.substring(1); + } + if (oldName.charAt(0) == '_') { + return "$" + oldName.substring(1); + } + return oldName; + } + + // TODO(bazel-team): implement attribute copy and other rule properties + + @SkylarkBuiltin(name = "rule", doc = + "Creates a new rule. Store it in a global value, so that it can be loaded and called " + + "from BUILD files.", + onlyLoadingPhase = true, + returnType = Function.class, + mandatoryParams = { + @Param(name = "implementation", type = UserDefinedFunction.class, + doc = "the function implementing this rule, has to have exactly one parameter: " + + "<code>ctx</code>. The function is called during analysis phase for each " + + "instance of the rule. It can access the attributes provided by the user. " + + "It must create actions to generate all the declared outputs.") + }, + optionalParams = { + @Param(name = "test", type = Boolean.class, doc = "Whether this rule is a test rule. " + + "If True, the rule must end with <code>_test</code> (otherwise it cannot)."), + @Param(name = "attrs", doc = + "dictionary to declare all the attributes of the rule. It maps from an attribute name " + + "to an attribute object (see 'attr' module). Attributes starting with <code>_</code> " + + "are private, and can be used to add an implicit dependency on a label."), + @Param(name = "outputs", doc = "outputs of this rule. " + + "It is a dictionary mapping from string to a template name. For example: " + + "<code>{\"ext\": \"${name}.ext\"}</code>. <br>" + // TODO(bazel-team): Make doc more clear, wrt late-bound attributes. + + "It may also be a function (which receives <code>ctx.attr</code> as argument) " + + "returning such a dictionary."), + @Param(name = "executable", type = Boolean.class, + doc = "whether this rule always outputs an executable of the same name or not. If True, " + + "there must be an action that generates <code>ctx.outputs.executable</code>.")}) + private static final SkylarkFunction rule = new SkylarkFunction("rule") { + + @Override + public Object call(Map<String, Object> arguments, FuncallExpression ast, + Environment funcallEnv) throws EvalException, ConversionException { + final Location loc = ast.getLocation(); + + RuleClassType type = RuleClassType.NORMAL; + if (arguments.containsKey("test") && EvalUtils.toBoolean(arguments.get("test"))) { + type = RuleClassType.TEST; + } + + // We'll set the name later, pass the empty string for now. + final RuleClass.Builder builder = type == RuleClassType.TEST + ? new RuleClass.Builder("", type, true, testBaseRule) + : new RuleClass.Builder("", type, true, baseRule); + + for (Map.Entry<String, Attribute.Builder> attr : castMap( + arguments.get("attrs"), String.class, Attribute.Builder.class, "attrs")) { + Attribute.Builder<?> attrBuilder = attr.getValue(); + String attrName = attributeToNative(attr.getKey(), loc, + attrBuilder.hasLateBoundValue()); + builder.addOrOverrideAttribute(attrBuilder.build(attrName)); + } + if (arguments.containsKey("executable") && (Boolean) arguments.get("executable")) { + builder.addOrOverrideAttribute( + attr("$is_executable", BOOLEAN).value(true) + .nonconfigurable("Called from RunCommand.isExecutable, which takes a Target") + .build()); + builder.setOutputsDefaultExecutable(); + } + + if (arguments.containsKey("outputs")) { + final Object implicitOutputs = arguments.get("outputs"); + if (implicitOutputs instanceof UserDefinedFunction) { + UserDefinedFunction func = (UserDefinedFunction) implicitOutputs; + final SkylarkCallbackFunction callback = + new SkylarkCallbackFunction(func, ast, (SkylarkEnvironment) funcallEnv); + builder.setImplicitOutputsFunction( + new SkylarkImplicitOutputsFunctionWithCallback(callback, loc)); + } else { + builder.setImplicitOutputsFunction(new SkylarkImplicitOutputsFunctionWithMap( + toMap(castMap(arguments.get("outputs"), String.class, String.class, + "implicit outputs of the rule class")))); + } + } + + builder.setConfiguredTargetFunction( + (UserDefinedFunction) arguments.get("implementation")); + builder.setRuleDefinitionEnvironment((SkylarkEnvironment) funcallEnv); + return new RuleFunction(builder, type); + } + }; + + // This class is needed for testing + static final class RuleFunction extends AbstractFunction { + // Note that this means that we can reuse the same builder. + // This is fine since we don't modify the builder from here. + private final RuleClass.Builder builder; + private final RuleClassType type; + + public RuleFunction(Builder builder, RuleClassType type) { + super("rule"); + this.builder = builder; + this.type = type; + } + + @Override + public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast, + Environment env) throws EvalException, InterruptedException { + try { + String ruleClassName = ast.getFunction().getName(); + if (ruleClassName.startsWith("_")) { + throw new EvalException(ast.getLocation(), "Invalid rule class name '" + ruleClassName + + "', cannot be private"); + } + if (type == RuleClassType.TEST != TargetUtils.isTestRuleName(ruleClassName)) { + throw new EvalException(ast.getLocation(), "Invalid rule class name '" + ruleClassName + + "', test rule class names must end with '_test' and other rule classes must not"); + } + RuleClass ruleClass = builder.build(ruleClassName); + PackageContext pkgContext = (PackageContext) env.lookup(PackageFactory.PKG_CONTEXT); + return RuleFactory.createAndAddRule(pkgContext, ruleClass, kwargs, ast); + } catch (InvalidRuleException | NameConflictException | NoSuchVariableException e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + } + + @VisibleForTesting + RuleClass.Builder getBuilder() { + return builder; + } + } + + @SkylarkBuiltin(name = "Label", doc = "Creates a Label referring to a BUILD target. Use " + + "this function only when you want to give a default value for the label attributes. " + + "Example: <br><pre class=language-python>Label(\"//tools:default\")</pre>", + returnType = Label.class, + mandatoryParams = {@Param(name = "label_string", type = String.class, + doc = "the label string")}) + private static final SkylarkFunction label = new SimpleSkylarkFunction("Label") { + @Override + public Object call(Map<String, Object> arguments, Location loc) throws EvalException, + ConversionException { + String labelString = (String) arguments.get("label_string"); + try { + return labelCache.get(labelString); + } catch (ExecutionException e) { + throw new EvalException(loc, "Illegal absolute label syntax: " + labelString); + } + } + }; + + @SkylarkBuiltin(name = "FileType", + doc = "Creates a file filter from a list of strings. For example, to match files ending " + + "with .cc or .cpp, use: <pre class=language-python>FileType([\".cc\", \".cpp\"])</pre>", + returnType = SkylarkFileType.class, + mandatoryParams = { + @Param(name = "types", type = SkylarkList.class, generic1 = String.class, + doc = "a list of the accepted file extensions")}) + private static final SkylarkFunction fileType = new SimpleSkylarkFunction("FileType") { + @Override + public Object call(Map<String, Object> arguments, Location loc) throws EvalException, + ConversionException { + return SkylarkFileType.of(castList(arguments.get("types"), String.class)); + } + }; + + @SkylarkBuiltin(name = "to_proto", + doc = "Creates a text message from the struct parameter. This method only works if all " + + "struct elements (recursively) are strings, ints, booleans, other structs or a " + + "list of these types. Quotes and new lines in strings are escaped. " + + "Examples:<br><pre class=language-python>" + + "struct(key=123).to_proto()\n# key: 123\n\n" + + "struct(key=True).to_proto()\n# key: true\n\n" + + "struct(key=[1, 2, 3]).to_proto()\n# key: 1\n# key: 2\n# key: 3\n\n" + + "struct(key='text').to_proto()\n# key: \"text\"\n\n" + + "struct(key=struct(inner_key='text')).to_proto()\n" + + "# key {\n# inner_key: \"text\"\n# }\n\n" + + "struct(key=[struct(inner_key=1), struct(inner_key=2)]).to_proto()\n" + + "# key {\n# inner_key: 1\n# }\n# key {\n# inner_key: 2\n# }\n\n" + + "struct(key=struct(inner_key=struct(inner_inner_key='text'))).to_proto()\n" + + "# key {\n# inner_key {\n# inner_inner_key: \"text\"\n# }\n# }\n</pre>", + objectType = SkylarkClassObject.class, returnType = String.class) + private static final SkylarkFunction toProto = new SimpleSkylarkFunction("to_proto") { + @Override + public Object call(Map<String, Object> arguments, Location loc) throws EvalException, + ConversionException { + ClassObject object = (ClassObject) arguments.get("self"); + StringBuilder sb = new StringBuilder(); + printTextMessage(object, sb, 0, loc); + return sb.toString(); + } + + private void printTextMessage(ClassObject object, StringBuilder sb, + int indent, Location loc) throws EvalException { + for (String key : object.getKeys()) { + printTextMessage(key, object.getValue(key), sb, indent, loc); + } + } + + private void printSimpleTextMessage(String key, Object value, StringBuilder sb, + int indent, Location loc, String container) throws EvalException { + if (value instanceof ClassObject) { + print(sb, key + " {", indent); + printTextMessage((ClassObject) value, sb, indent + 1, loc); + print(sb, "}", indent); + } else if (value instanceof String) { + print(sb, key + ": \"" + escape((String) value) + "\"", indent); + } else if (value instanceof Integer) { + print(sb, key + ": " + value, indent); + } else if (value instanceof Boolean) { + // We're relying on the fact that Java converts Booleans to Strings in the same way + // as the protocol buffers do. + print(sb, key + ": " + value, indent); + } else { + throw new EvalException(loc, + "Invalid text format, expected a struct, a string, a bool, or an int but got a " + + EvalUtils.getDatatypeName(value) + " for " + container + " '" + key + "'"); + } + } + + private void printTextMessage(String key, Object value, StringBuilder sb, + int indent, Location loc) throws EvalException { + if (value instanceof SkylarkList) { + for (Object item : ((SkylarkList) value)) { + // TODO(bazel-team): There should be some constraint on the fields of the structs + // in the same list but we ignore that for now. + printSimpleTextMessage(key, item, sb, indent, loc, "list element in struct field"); + } + } else { + printSimpleTextMessage(key, value, sb, indent, loc, "struct field"); + } + } + + private String escape(String string) { + // TODO(bazel-team): use guava's SourceCodeEscapers when it's released. + return string.replace("\"", "\\\"").replace("\n", "\\n"); + } + + private void print(StringBuilder sb, String text, int indent) { + for (int i = 0; i < indent; i++) { + sb.append(" "); + } + sb.append(text); + sb.append("\n"); + } + }; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleConfiguredTargetBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleConfiguredTargetBuilder.java new file mode 100644 index 0000000000..528e0f1dc8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleConfiguredTargetBuilder.java @@ -0,0 +1,213 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules; + +import static com.google.devtools.build.lib.syntax.SkylarkFunction.cast; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Function; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.syntax.SkylarkNestedSet; + +/** + * A helper class to build Rule Configured Targets via runtime loaded rule implementations + * defined using the Skylark Build Extension Language. This is experimental code. + */ +public final class SkylarkRuleConfiguredTargetBuilder { + + /** + * Create a Rule Configured Target from the ruleContext and the ruleImplementation. + */ + public static ConfiguredTarget buildRule(RuleContext ruleContext, + Function ruleImplementation) { + String expectError = ruleContext.attributes().get("expect_failure", Type.STRING); + try { + SkylarkRuleContext skylarkRuleContext = new SkylarkRuleContext(ruleContext); + SkylarkEnvironment env = ruleContext.getRule().getRuleClassObject() + .getRuleDefinitionEnvironment().cloneEnv( + ruleContext.getAnalysisEnvironment().getEventHandler()); + // Collect the symbols to disable statically and pass at the next call, so we don't need to + // clone the RuleDefinitionEnvironment. + env.disableOnlyLoadingPhaseObjects(); + Object target = ruleImplementation.call(ImmutableList.<Object>of(skylarkRuleContext), + ImmutableMap.<String, Object>of(), null, env); + + if (ruleContext.hasErrors()) { + return null; + } else if (!(target instanceof SkylarkClassObject) && target != Environment.NONE) { + ruleContext.ruleError("Rule implementation doesn't return a struct"); + return null; + } else if (!expectError.isEmpty()) { + ruleContext.ruleError("Expected error not found: " + expectError); + return null; + } + ConfiguredTarget configuredTarget = createTarget(ruleContext, target); + checkOrphanArtifacts(ruleContext); + return configuredTarget; + + } catch (InterruptedException e) { + ruleContext.ruleError(e.getMessage()); + return null; + } catch (EvalException e) { + // If the error was expected, return an empty target. + if (!expectError.isEmpty() && e.getMessage().matches(expectError)) { + return new com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder(ruleContext) + .add(RunfilesProvider.class, RunfilesProvider.EMPTY) + .build(); + } + ruleContext.ruleError("\n" + e.print()); + return null; + } + } + + private static void checkOrphanArtifacts(RuleContext ruleContext) throws EvalException { + ImmutableSet<Artifact> orphanArtifacts = + ruleContext.getAnalysisEnvironment().getOrphanArtifacts(); + if (!orphanArtifacts.isEmpty()) { + throw new EvalException(null, "The following files have no generating action:\n" + + Joiner.on("\n").join(Iterables.transform(orphanArtifacts, + new com.google.common.base.Function<Artifact, String>() { + @Override + public String apply(Artifact artifact) { + return artifact.getRootRelativePathString(); + } + }))); + } + } + + // TODO(bazel-team): this whole defaulting - overriding executable, runfiles and files_to_build + // is getting out of hand. Clean this whole mess up. + private static ConfiguredTarget createTarget(RuleContext ruleContext, Object target) + throws EvalException { + Artifact executable = getExecutable(ruleContext, target); + RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(ruleContext); + // Set the default files to build. + NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.<Artifact>stableOrder() + .addAll(ruleContext.getOutputArtifacts()); + if (executable != null) { + filesToBuild.add(executable); + } + builder.setFilesToBuild(filesToBuild.build()); + return addStructFields(ruleContext, builder, target, executable); + } + + private static Artifact getExecutable(RuleContext ruleContext, Object target) + throws EvalException { + Artifact executable = ruleContext.getRule().getRuleClassObject().outputsDefaultExecutable() + // This doesn't actually create a new Artifact just returns the one + // created in SkylarkruleContext. + ? ruleContext.createOutputArtifact() : null; + if (target instanceof SkylarkClassObject) { + SkylarkClassObject struct = (SkylarkClassObject) target; + if (struct.getValue("executable") != null) { + // We need this because of genrule.bzl. This overrides the default executable. + executable = cast( + struct.getValue("executable"), Artifact.class, "executable", struct.getCreationLoc()); + } + } + return executable; + } + + private static ConfiguredTarget addStructFields(RuleContext ruleContext, + RuleConfiguredTargetBuilder builder, Object target, Artifact executable) + throws EvalException { + Location loc = null; + Runfiles statelessRunfiles = null; + Runfiles dataRunfiles = null; + Runfiles defaultRunfiles = null; + if (target instanceof SkylarkClassObject) { + SkylarkClassObject struct = (SkylarkClassObject) target; + loc = struct.getCreationLoc(); + for (String key : struct.getKeys()) { + if (key.equals("files")) { + // If we specify files_to_build we don't have the executable in it by default. + builder.setFilesToBuild(cast(struct.getValue("files"), + SkylarkNestedSet.class, "files", loc).getSet(Artifact.class)); + } else if (key.equals("runfiles")) { + statelessRunfiles = cast(struct.getValue("runfiles"), Runfiles.class, "runfiles", loc); + } else if (key.equals("data_runfiles")) { + dataRunfiles = + cast(struct.getValue("data_runfiles"), Runfiles.class, "data_runfiles", loc); + } else if (key.equals("default_runfiles")) { + defaultRunfiles = + cast(struct.getValue("default_runfiles"), Runfiles.class, "default_runfiles", loc); + } else if (!key.equals("executable")) { + // We handled executable already. + builder.addSkylarkTransitiveInfo(key, struct.getValue(key), loc); + } + } + } + + if ((statelessRunfiles != null) && (dataRunfiles != null || defaultRunfiles != null)) { + throw new EvalException(loc, "Cannot specify the provider 'runfiles' " + + "together with 'data_runfiles' or 'default_runfiles'"); + } + + if (statelessRunfiles == null && dataRunfiles == null && defaultRunfiles == null) { + // No runfiles specified, set default + statelessRunfiles = Runfiles.EMPTY; + } + + RunfilesProvider runfilesProvider = statelessRunfiles != null + ? RunfilesProvider.simple(merge(statelessRunfiles, executable)) + : RunfilesProvider.withData( + // The executable doesn't get into the default runfiles if we have runfiles states. + // This is to keep skylark genrule consistent with the original genrule. + defaultRunfiles != null ? defaultRunfiles : Runfiles.EMPTY, + dataRunfiles != null ? dataRunfiles : Runfiles.EMPTY); + builder.addProvider(RunfilesProvider.class, runfilesProvider); + + Runfiles computedDefaultRunfiles = runfilesProvider.getDefaultRunfiles(); + // This works because we only allowed to call a rule *_test iff it's a test type rule. + boolean testRule = TargetUtils.isTestRuleName(ruleContext.getRule().getRuleClass()); + if (testRule && computedDefaultRunfiles.isEmpty()) { + throw new EvalException(loc, "Test rules have to define runfiles"); + } + if (executable != null || testRule) { + RunfilesSupport runfilesSupport = computedDefaultRunfiles.isEmpty() + ? null : RunfilesSupport.withExecutable(ruleContext, computedDefaultRunfiles, executable); + builder.setRunfilesSupport(runfilesSupport, executable); + } + try { + return builder.build(); + } catch (IllegalArgumentException e) { + throw new EvalException(loc, e.getMessage()); + } + } + + private static Runfiles merge(Runfiles runfiles, Artifact executable) { + if (executable == null) { + return runfiles; + } + return new Runfiles.Builder().addArtifact(executable).merge(runfiles).build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleContext.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleContext.java new file mode 100644 index 0000000000..fc06677fdc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleContext.java @@ -0,0 +1,484 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules; + +import com.google.common.base.Preconditions; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.ConfigurationMakeVariableContext; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.LabelExpander; +import com.google.devtools.build.lib.analysis.LabelExpander.NotUniqueExpansionException; +import com.google.devtools.build.lib.analysis.MakeVariableExpander.ExpansionException; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SkylarkImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.RawAttributeMapper; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.shell.ShellUtils; +import com.google.devtools.build.lib.shell.ShellUtils.TokenizationException; +import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.FuncallExpression.FuncallException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkList; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.syntax.SkylarkType; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A Skylark API for the ruleContext. + */ +@SkylarkModule(name = "ctx", doc = "The context of the rule containing helper functions and " + + "information about attributes, depending targets and outputs. " + + "You get a ctx object as an argument to the <code>implementation</code> function when " + + "you create a rule.") +public final class SkylarkRuleContext { + + public static final String PROVIDER_CLASS_PREFIX = "com.google.devtools.build.lib."; + + static final LoadingCache<String, Class<?>> classCache = CacheBuilder.newBuilder() + .initialCapacity(10) + .maximumSize(100) + .build(new CacheLoader<String, Class<?>>() { + + @Override + public Class<?> load(String key) throws Exception { + String classPath = SkylarkRuleContext.PROVIDER_CLASS_PREFIX + key; + return Class.forName(classPath); + } + }); + + private final RuleContext ruleContext; + + // TODO(bazel-team): support configurable attributes. + private final SkylarkClassObject attrObject; + + private final SkylarkClassObject outputsObject; + + private final SkylarkClassObject executableObject; + + private final SkylarkClassObject fileObject; + + private final SkylarkClassObject filesObject; + + private final SkylarkClassObject targetsObject; + + private final SkylarkClassObject targetObject; + + // TODO(bazel-team): we only need this because of the css_binary rule. + private final ImmutableMap<Artifact, Label> artifactLabelMap; + + private final ImmutableMap<Artifact, FilesToRunProvider> executableRunfilesMap; + + /** + * In native code, private values start with $. + * In Skylark, private values start with _, because of the grammar. + */ + private String attributeToSkylark(String oldName) { + if (!oldName.isEmpty() && (oldName.charAt(0) == '$' || oldName.charAt(0) == ':')) { + return "_" + oldName.substring(1); + } + return oldName; + } + + /** + * Creates a new SkylarkRuleContext using ruleContext. + */ + public SkylarkRuleContext(RuleContext ruleContext) throws EvalException { + this.ruleContext = Preconditions.checkNotNull(ruleContext); + + HashMap<String, Object> outputsBuilder = new HashMap<>(); + if (ruleContext.getRule().getRuleClassObject().outputsDefaultExecutable()) { + addOutput(outputsBuilder, "executable", ruleContext.createOutputArtifact()); + } + ImplicitOutputsFunction implicitOutputsFunction = + ruleContext.getRule().getRuleClassObject().getImplicitOutputsFunction(); + + if (implicitOutputsFunction instanceof SkylarkImplicitOutputsFunction) { + SkylarkImplicitOutputsFunction func = (SkylarkImplicitOutputsFunction) + ruleContext.getRule().getRuleClassObject().getImplicitOutputsFunction(); + for (Map.Entry<String, String> entry : func.calculateOutputs( + RawAttributeMapper.of(ruleContext.getRule())).entrySet()) { + addOutput(outputsBuilder, entry.getKey(), + ruleContext.getImplicitOutputArtifact(entry.getValue())); + } + } + + ImmutableMap.Builder<Artifact, Label> artifactLabelMapBuilder = + ImmutableMap.builder(); + for (Attribute a : ruleContext.getRule().getAttributes()) { + String attrName = a.getName(); + Type<?> type = a.getType(); + if (type != Type.OUTPUT && type != Type.OUTPUT_LIST) { + continue; + } + ImmutableList.Builder<Artifact> artifactsBuilder = ImmutableList.builder(); + for (OutputFile outputFile : ruleContext.getRule().getOutputFileMap().get(attrName)) { + Artifact artifact = ruleContext.createOutputArtifact(outputFile); + artifactsBuilder.add(artifact); + artifactLabelMapBuilder.put(artifact, outputFile.getLabel()); + } + ImmutableList<Artifact> artifacts = artifactsBuilder.build(); + + if (type == Type.OUTPUT) { + if (artifacts.size() == 1) { + addOutput(outputsBuilder, attrName, Iterables.getOnlyElement(artifacts)); + } else { + addOutput(outputsBuilder, attrName, Environment.NONE); + } + } else if (type == Type.OUTPUT_LIST) { + addOutput(outputsBuilder, attrName, + SkylarkList.list(artifacts, Artifact.class)); + } else { + throw new IllegalArgumentException( + "Type of " + attrName + "(" + type + ") is not output type "); + } + } + artifactLabelMap = artifactLabelMapBuilder.build(); + outputsObject = new SkylarkClassObject(outputsBuilder, "No such output '%s'"); + + ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>(); + ImmutableMap.Builder<String, Object> executableBuilder = new ImmutableMap.Builder<>(); + ImmutableMap.Builder<Artifact, FilesToRunProvider> executableRunfilesbuilder = + new ImmutableMap.Builder<>(); + ImmutableMap.Builder<String, Object> fileBuilder = new ImmutableMap.Builder<>(); + ImmutableMap.Builder<String, Object> filesBuilder = new ImmutableMap.Builder<>(); + ImmutableMap.Builder<String, Object> targetBuilder = new ImmutableMap.Builder<>(); + ImmutableMap.Builder<String, Object> targetsBuilder = new ImmutableMap.Builder<>(); + for (Attribute a : ruleContext.getRule().getAttributes()) { + Type<?> type = a.getType(); + Object val = ruleContext.attributes().get(a.getName(), type); + builder.put(attributeToSkylark(a.getName()), val == null ? Environment.NONE + // Attribute values should be type safe + : SkylarkType.convertToSkylark(val, null)); + if (type != Type.LABEL && type != Type.LABEL_LIST) { + continue; + } + String skyname = attributeToSkylark(a.getName()); + Mode mode = getMode(a.getName()); + if (a.isExecutable()) { + // In Skylark only label (not label list) type attributes can have the Executable flag. + FilesToRunProvider provider = ruleContext.getExecutablePrerequisite(a.getName(), mode); + if (provider != null && provider.getExecutable() != null) { + Artifact executable = provider.getExecutable(); + executableBuilder.put(skyname, executable); + executableRunfilesbuilder.put(executable, provider); + } else { + executableBuilder.put(skyname, Environment.NONE); + } + } + if (a.isSingleArtifact()) { + // In Skylark only label (not label list) type attributes can have the SingleArtifact flag. + Artifact artifact = ruleContext.getPrerequisiteArtifact(a.getName(), mode); + if (artifact != null) { + fileBuilder.put(skyname, artifact); + } else { + fileBuilder.put(skyname, Environment.NONE); + } + } + filesBuilder.put(skyname, ruleContext.getPrerequisiteArtifacts(a.getName(), mode).list()); + targetsBuilder.put(skyname, SkylarkList.list( + ruleContext.getPrerequisites(a.getName(), mode), TransitiveInfoCollection.class)); + if (type == Type.LABEL) { + Object prereq = ruleContext.getPrerequisite(a.getName(), mode); + if (prereq != null) { + targetBuilder.put(skyname, prereq); + } else { + targetBuilder.put(skyname, Environment.NONE); + } + } + } + attrObject = new SkylarkClassObject(builder.build(), "No such attribute '%s'"); + executableObject = new SkylarkClassObject(executableBuilder.build(), "No such executable. " + + "Make sure there is a '%s' label type attribute marked as 'executable'"); + fileObject = new SkylarkClassObject(fileBuilder.build(), + "No such file. Make sure there is a '%s' label type attribute marked as 'single_file'"); + filesObject = new SkylarkClassObject(filesBuilder.build(), + "No such files. Make sure there is a '%s' label or label_list type attribute"); + targetObject = new SkylarkClassObject(targetBuilder.build(), + "No such target. Make sure there is a '%s' label type attribute"); + targetsObject = new SkylarkClassObject(targetsBuilder.build(), + "No such targets. Make sure there is a '%s' label or label_list type attribute"); + executableRunfilesMap = executableRunfilesbuilder.build(); + } + + private void addOutput(HashMap<String, Object> outputsBuilder, String key, Object value) + throws EvalException { + if (outputsBuilder.containsKey(key)) { + throw new EvalException(null, "Multiple outputs with the same key: " + key); + } + outputsBuilder.put(key, value); + } + + /** + * Returns the original ruleContext. + */ + public RuleContext getRuleContext() { + return ruleContext; + } + + private Mode getMode(String attributeName) { + return ruleContext.getAttributeMode(attributeName); + } + + @SkylarkCallable(name = "attr", structField = true, + doc = "A struct to access the values of the attributes. The values are provided by " + + "the user (if not, a default value is used).") + public SkylarkClassObject getAttr() { + return attrObject; + } + + /** + * <p>See {@link RuleContext#getExecutablePrerequisite(String, Mode)}. + */ + @SkylarkCallable(name = "executable", structField = true, + doc = "A <code>struct</code> containing executable files defined in label type " + + "attributes marked as <code>executable=True</code>. The struct fields correspond " + + "to the attribute names. The struct value is always a <code>file</code>s or " + + "<code>None</code>. If a non-mandatory attribute is not specified in the rule " + + "the corresponding struct value is <code>None</code>. If a label type is not " + + "marked as <code>executable=True</code>, no corresponding struct field is generated.") + public SkylarkClassObject getExecutable() { + return executableObject; + } + + /** + * See {@link RuleContext#getPrerequisiteArtifact(String, Mode)}. + */ + @SkylarkCallable(name = "file", structField = true, + doc = "A <code>struct</code> containing files defined in label type " + + "attributes marked as <code>single_file=True</code>. The struct fields correspond " + + "to the attribute names. The struct value is always a <code>file</code> or " + + "<code>None</code>. If a non-mandatory attribute is not specified in the rule " + + "the corresponding struct value is <code>None</code>. If a label type is not " + + "marked as <code>single_file=True</code>, no corresponding struct field is generated.") + public SkylarkClassObject getFile() { + return fileObject; + } + + /** + * See {@link RuleContext#getPrerequisiteArtifacts(String, Mode)}. + */ + @SkylarkCallable(name = "files", structField = true, + doc = "A <code>struct</code> containing files defined in label or label list " + + "type attributes. The struct fields correspond to the attribute names. The struct " + + "values are <code>list</code> of <code>file</code>s. If a non-mandatory attribute is " + + "not specified in the rule, an empty list is generated.") + public SkylarkClassObject getFiles() { + return filesObject; + } + + /** + * See {@link RuleContext#getPrerequisite(String, Mode)}. + */ + @SkylarkCallable(name = "target", structField = true, + doc = "A <code>struct</code> containing prerequisite targets defined in label type " + + "attributes. The struct fields correspond to the attribute names. The struct value " + + "is always a <code>target</code> or <code>None</code>. If a non-mandatory attribute " + + "is not specified in the rule, the corresponding struct value is <code>None</code>.") + public SkylarkClassObject getTarget() { + return targetObject; + } + + /** + * See {@link RuleContext#getPrerequisites(String, Mode)}. + */ + @SkylarkCallable(name = "targets", structField = true, + doc = "A <code>struct</code> containing prerequisite targets defined in label or label list " + + "type attributes. The struct fields correspond to the attribute names. The struct " + + "values are <code>list</code> of <code>target</code>s. If a non-mandatory attribute is " + + "not specified in the rule, an empty list is generated.") + public SkylarkClassObject getTargets() { + return targetsObject; + } + + @SkylarkCallable(name = "label", structField = true, doc = "The label of this rule.") + public Label getLabel() { + return ruleContext.getLabel(); + } + + @SkylarkCallable(name = "configuration", structField = true, + doc = "Returns the default configuration. See the <code>configuration</code> type for " + + "more details.") + public BuildConfiguration getConfiguration() { + return ruleContext.getConfiguration(); + } + + @SkylarkCallable(name = "host_configuration", structField = true, + doc = "Returns the host configuration. See the <code>configuration</code> type for " + + "more details.") + public BuildConfiguration getHostConfiguration() { + return ruleContext.getHostConfiguration(); + } + + @SkylarkCallable(name = "data_configuration", structField = true, + doc = "Returns the data configuration. See the <code>configuration</code> type for " + + "more details.") + public BuildConfiguration getDataConfiguration() { + return ruleContext.getConfiguration().getConfiguration(ConfigurationTransition.DATA); + } + + @SkylarkCallable(structField = true, + doc = "A <code>struct</code> containing all the output files." + + " The struct is generated the following way:<br>" + + "<ul><li>If the rule is marked as <code>executable=True</code> the struct has an " + + "\"executable\" field with the rules default executable <code>file</code> value." + + "<li>For every entry in the rule's <code>outputs</code> dict a field is generated with " + + "the same name and the corresponding <code>file</code> value." + + "<li>For every output type attribute a struct field is generated with the " + + "same name and the corresponding <code>file</code> value or <code>None</code>, " + + "if no value is specified in the rule." + + "<li>For every output list type attribute a struct field is generated with the " + + "same name and corresponding <code>list</code> of <code>file</code>s value " + + "(an empty list if no value is specified in the rule.</ul>") + public SkylarkClassObject outputs() { + return outputsObject; + } + + @Override + public String toString() { + return ruleContext.getLabel().toString(); + } + + @SkylarkCallable(doc = "Splits a shell command to a list of tokens.", hidden = true) + public List<String> tokenize(String optionString) throws FuncallException { + List<String> options = new ArrayList<String>(); + try { + ShellUtils.tokenize(options, optionString); + } catch (TokenizationException e) { + throw new FuncallException(e.getMessage() + " while tokenizing '" + optionString + "'"); + } + return ImmutableList.copyOf(options); + } + + @SkylarkCallable(doc = + "Expands all references to labels embedded within a string for all files using a mapping " + + "from definition labels (i.e. the label in the output type attribute) to files. Deprecated.", + hidden = true) + public String expand(@Nullable String expression, + List<Artifact> artifacts, Label labelResolver) throws FuncallException { + try { + Map<Label, Iterable<Artifact>> labelMap = new HashMap<>(); + for (Artifact artifact : artifacts) { + labelMap.put(artifactLabelMap.get(artifact), ImmutableList.of(artifact)); + } + return LabelExpander.expand(expression, labelMap, labelResolver); + } catch (NotUniqueExpansionException e) { + throw new FuncallException(e.getMessage() + " while expanding '" + expression + "'"); + } + } + + @SkylarkCallable(doc = + "Creates a file with the given filename. You must create an action that generates " + + "the file. If the file should be publicly visible, declare a rule " + + "output instead when possible.") + public Artifact newFile(Root root, String filename) { + PathFragment fragment = ruleContext.getLabel().getPackageFragment(); + for (String pathFragmentString : filename.split("/")) { + fragment = fragment.getRelative(pathFragmentString); + } + return ruleContext.getAnalysisEnvironment().getDerivedArtifact(fragment, root); + } + + @SkylarkCallable(doc = + "Creates a new file, derived from the given file and suffix. " + + "You must create an action that generates " + + "the file. If the file should be publicly visible, declare a rule " + + "output instead when possible.") + public Artifact newFile(Root root, Artifact baseArtifact, String suffix) { + PathFragment original = baseArtifact.getRootRelativePath(); + PathFragment fragment = original.replaceName(original.getBaseName() + suffix); + return ruleContext.getAnalysisEnvironment().getDerivedArtifact(fragment, root); + } + + @SkylarkCallable(doc = "", hidden = true) + public NestedSet<Artifact> middleMan(String attribute) { + return AnalysisUtils.getMiddlemanFor(ruleContext, attribute); + } + + @SkylarkCallable(doc = "", hidden = true) + public boolean checkPlaceholders(String template, List<String> allowedPlaceholders) { + List<String> actualPlaceHolders = new LinkedList<>(); + Set<String> allowedPlaceholderSet = ImmutableSet.copyOf(allowedPlaceholders); + ImplicitOutputsFunction.createPlaceholderSubstitutionFormatString(template, actualPlaceHolders); + for (String placeholder : actualPlaceHolders) { + if (!allowedPlaceholderSet.contains(placeholder)) { + return false; + } + } + return true; + } + + @SkylarkCallable(doc = "") + public String expandMakeVariables(String attributeName, String command, + final Map<String, String> additionalSubstitutions) { + return ruleContext.expandMakeVariables(attributeName, + command, new ConfigurationMakeVariableContext(ruleContext.getRule().getPackage(), + ruleContext.getConfiguration()) { + @Override + public String lookupMakeVariable(String name) throws ExpansionException { + if (additionalSubstitutions.containsKey(name)) { + return additionalSubstitutions.get(name); + } else { + return super.lookupMakeVariable(name); + } + } + }); + } + + FilesToRunProvider getExecutableRunfiles(Artifact executable) { + return executableRunfilesMap.get(executable); + } + + @SkylarkCallable(name = "info_file", structField = true, hidden = true, + doc = "Returns the file that is used to hold the non-volatile workspace status for the " + + "current build request.") + public Artifact getStableWorkspaceStatus() { + return ruleContext.getAnalysisEnvironment().getStableWorkspaceStatusArtifact(); + } + + @SkylarkCallable(name = "version_file", structField = true, hidden = true, + doc = "Returns the file that is used to hold the volatile workspace status for the " + + "current build request.") + public Artifact getVolatileWorkspaceStatus() { + return ruleContext.getAnalysisEnvironment().getVolatileWorkspaceStatusArtifact(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java new file mode 100644 index 0000000000..1f7d1609b0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java @@ -0,0 +1,367 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.CommandHelper; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.MakeVariableExpander; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction; +import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.Substitution; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.Type.ConversionException; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.EvalUtils; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param; +import com.google.devtools.build.lib.syntax.SkylarkFunction; +import com.google.devtools.build.lib.syntax.SkylarkFunction.SimpleSkylarkFunction; +import com.google.devtools.build.lib.syntax.SkylarkList; +import com.google.devtools.build.lib.syntax.SkylarkNestedSet; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Map; +import java.util.concurrent.ExecutionException; + +// TODO(bazel-team): function argument names are often duplicated, +// figure out a nicely readable way to get rid of the duplications. +/** + * A helper class to provide an easier API for Skylark rule implementations + * and hide the original Java API. This is experimental code. + */ +public class SkylarkRuleImplementationFunctions { + + // TODO(bazel-team): add all the remaining parameters + // TODO(bazel-team): merge executable and arguments + /** + * A Skylark built-in function to create and register a SpawnAction using a + * dictionary of parameters: + * createSpawnAction( + * inputs = [input1, input2, ...], + * outputs = [output1, output2, ...], + * executable = executable, + * arguments = [argument1, argument2, ...], + * mnemonic = 'mnemonic', + * command = 'command', + * register = 1 + * ) + */ + @SkylarkBuiltin(name = "action", + doc = "Creates an action that runs an executable or a shell command.", + objectType = SkylarkRuleContext.class, + returnType = Environment.NoneType.class, + mandatoryParams = { + @Param(name = "outputs", type = SkylarkList.class, generic1 = Artifact.class, + doc = "list of the output files of the action")}, + optionalParams = { + @Param(name = "inputs", type = SkylarkList.class, generic1 = Artifact.class, + doc = "list of the input files of the action"), + @Param(name = "executable", doc = "the executable file to be called by the action"), + @Param(name = "arguments", type = SkylarkList.class, generic1 = String.class, + doc = "command line arguments of the action"), + @Param(name = "mnemonic", type = String.class, doc = "mnemonic"), + @Param(name = "command", doc = "shell command to execute"), + @Param(name = "command_line", doc = "a command line to execute"), + @Param(name = "progress_message", type = String.class, + doc = "progress message to show to the user during the build"), + @Param(name = "use_default_shell_env", type = Boolean.class, + doc = "whether the action should use the built in shell environment or not"), + @Param(name = "env", type = Map.class, doc = "sets the dictionary of environment variables"), + @Param(name = "execution_requirements", type = Map.class, + doc = "information for scheduling the action"), + @Param(name = "input_manifests", type = Map.class, + doc = "sets the map of input manifests files; " + + "they are typicially generated by the command_helper")}) + private static final SkylarkFunction createSpawnAction = + new SimpleSkylarkFunction("action") { + + @Override + public Object call(Map<String, Object> params, Location loc) throws EvalException, + ConversionException { + SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self"); + SpawnAction.Builder builder = new SpawnAction.Builder(); + // TODO(bazel-team): builder still makes unnecessary copies of inputs, outputs and args. + builder.addInputs(castList(params.get("inputs"), Artifact.class)); + builder.addOutputs(castList(params.get("outputs"), Artifact.class)); + builder.addArguments(castList(params.get("arguments"), String.class)); + if (params.containsKey("executable")) { + Object exe = params.get("executable"); + if (exe instanceof Artifact) { + Artifact executable = (Artifact) exe; + builder.addInput(executable); + FilesToRunProvider provider = ctx.getExecutableRunfiles(executable); + if (provider == null) { + builder.setExecutable((Artifact) exe); + } else { + builder.setExecutable(provider); + } + } else if (exe instanceof PathFragment) { + builder.setExecutable((PathFragment) exe); + } else { + throw new EvalException(loc, "expected file or PathFragment for " + + "executable but got " + EvalUtils.getDatatypeName(exe) + " instead"); + } + } + if (params.containsKey("command") == params.containsKey("executable")) { + throw new EvalException(loc, "You must specify either 'command' or 'executable' argument"); + } + if (params.containsKey("command")) { + Object command = params.get("command"); + if (command instanceof String) { + builder.setShellCommand((String) command); + } else if (command instanceof SkylarkList) { + SkylarkList commandList = (SkylarkList) command; + if (commandList.size() < 3) { + throw new EvalException(loc, "'command' list has to be of size at least 3"); + } + builder.setShellCommand(castList(commandList, String.class, "command")); + } else { + throw new EvalException(loc, "expected string or list of strings for " + + "command instead of " + EvalUtils.getDatatypeName(command)); + } + } + if (params.containsKey("command_line")) { + builder.setCommandLine(CommandLine.ofCharSequences(ImmutableList.copyOf(castList( + params.get("command_line"), CharSequence.class, "command line")))); + } + if (params.containsKey("mnemonic")) { + builder.setMnemonic((String) params.get("mnemonic")); + } + if (params.containsKey("env")) { + builder.setEnvironment( + toMap(castMap(params.get("env"), String.class, String.class, "env"))); + } + if (params.containsKey("progress_message")) { + builder.setProgressMessage((String) params.get("progress_message")); + } + if (params.containsKey("use_default_shell_env") + && EvalUtils.toBoolean(params.get("use_default_shell_env"))) { + builder.useDefaultShellEnvironment(); + } + if (params.containsKey("execution_requirements")) { + builder.setExecutionInfo(toMap(castMap(params.get("execution_requirements"), + String.class, String.class, "execution_requirements"))); + } + if (params.containsKey("input_manifests")) { + for (Map.Entry<PathFragment, Artifact> entry : castMap(params.get("input_manifests"), + PathFragment.class, Artifact.class, "input manifest file map")) { + builder.addInputManifest(entry.getValue(), entry.getKey()); + } + } + // Always register the action + ctx.getRuleContext().registerAction(builder.build(ctx.getRuleContext())); + return Environment.NONE; + } + }; + + // TODO(bazel-team): improve this method to be more memory friendly + @SkylarkBuiltin(name = "file_action", + doc = "Creates a file write action.", + objectType = SkylarkRuleContext.class, + returnType = Environment.NoneType.class, + optionalParams = { + @Param(name = "executable", type = Boolean.class, + doc = "whether the output file should be executable (default is False)"), + }, + mandatoryParams = { + @Param(name = "output", type = Artifact.class, doc = "the output file"), + @Param(name = "content", type = String.class, doc = "the contents of the file")}) + private static final SkylarkFunction createFileWriteAction = + new SimpleSkylarkFunction("file_action") { + + @Override + public Object call(Map<String, Object> params, Location loc) throws EvalException, + ConversionException { + SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self"); + boolean executable = params.containsKey("executable") && (Boolean) params.get("executable"); + FileWriteAction action = new FileWriteAction( + ctx.getRuleContext().getActionOwner(), + (Artifact) params.get("output"), + (String) params.get("content"), + executable); + ctx.getRuleContext().registerAction(action); + return action; + } + }; + + @SkylarkBuiltin(name = "template_action", + doc = "Creates a template expansion action.", + objectType = SkylarkRuleContext.class, + returnType = Environment.NoneType.class, + mandatoryParams = { + @Param(name = "template", type = Artifact.class, doc = "the template file"), + @Param(name = "output", type = Artifact.class, doc = "the output file"), + @Param(name = "substitutions", type = Map.class, + doc = "substitutions to make when expanding the template")}, + optionalParams = { + @Param(name = "executable", type = Boolean.class, + doc = "whether the output file should be executable (default is False)")}) + private static final SkylarkFunction createTemplateAction = + new SimpleSkylarkFunction("template_action") { + + @Override + public Object call(Map<String, Object> params, Location loc) throws EvalException, + ConversionException { + SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self"); + ImmutableList.Builder<Substitution> substitutions = ImmutableList.builder(); + for (Map.Entry<String, String> substitution + : castMap(params.get("substitutions"), String.class, String.class, "substitutions")) { + substitutions.add(Substitution.of(substitution.getKey(), substitution.getValue())); + } + + boolean executable = params.containsKey("executable") && (Boolean) params.get("executable"); + TemplateExpansionAction action = new TemplateExpansionAction( + ctx.getRuleContext().getActionOwner(), + (Artifact) params.get("template"), + (Artifact) params.get("output"), + substitutions.build(), + executable); + ctx.getRuleContext().registerAction(action); + return action; + } + }; + + /** + * A built in Skylark helper function to access the + * Transitive info providers of Transitive info collections. + */ + @SkylarkBuiltin(name = "provider", + doc = "Returns the transitive info provider provided by the target.", + mandatoryParams = { + @Param(name = "target", type = TransitiveInfoCollection.class, + doc = "the configured target which provides the provider"), + @Param(name = "type", type = String.class, doc = "the class type of the provider")}) + private static final SkylarkFunction provider = new SimpleSkylarkFunction("provider") { + @Override + public Object call(Map<String, Object> params, Location loc) throws EvalException { + TransitiveInfoCollection target = (TransitiveInfoCollection) params.get("target"); + String type = (String) params.get("type"); + try { + Class<?> classType = SkylarkRuleContext.classCache.get(type); + Class<? extends TransitiveInfoProvider> convertedClass = + classType.asSubclass(TransitiveInfoProvider.class); + Object result = target.getProvider(convertedClass); + return result == null ? Environment.NONE : result; + } catch (ExecutionException e) { + throw new EvalException(loc, "Unknown class type " + type); + } catch (ClassCastException e) { + throw new EvalException(loc, "Not a TransitiveInfoProvider " + type); + } + } + }; + + // TODO(bazel-team): Remove runfile states from Skylark. + @SkylarkBuiltin(name = "runfiles", + doc = "Creates a runfiles object.", + objectType = SkylarkRuleContext.class, + returnType = Runfiles.class, + optionalParams = { + @Param(name = "files", type = SkylarkList.class, generic1 = Artifact.class, + doc = "The list of files to be added to the runfiles."), + // TODO(bazel-team): If we have a memory efficient support for lazy list containing NestedSets + // we can remove this and just use files = [file] + list(set) + @Param(name = "transitive_files", type = SkylarkNestedSet.class, generic1 = Artifact.class, + doc = "The (transitive) set of files to be added to the runfiles."), + @Param(name = "collect_data", type = Boolean.class, doc = "Whether to collect the data " + + "runfiles from the dependencies in srcs, data and deps attributes."), + @Param(name = "collect_default", type = Boolean.class, doc = "Whether to collect the default " + + "runfiles from the dependencies in srcs, data and deps attributes.")}) + private static final SkylarkFunction runfiles = new SimpleSkylarkFunction("runfiles") { + @Override + public Object call(Map<String, Object> params, Location loc) throws EvalException, + ConversionException { + SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self"); + Runfiles.Builder builder = new Runfiles.Builder(); + if (params.containsKey("collect_data") && (Boolean) params.get("collect_data")) { + builder.addRunfiles(ctx.getRuleContext(), RunfilesProvider.DATA_RUNFILES); + } + if (params.containsKey("collect_default") && (Boolean) params.get("collect_default")) { + builder.addRunfiles(ctx.getRuleContext(), RunfilesProvider.DEFAULT_RUNFILES); + } + if (params.containsKey("files")) { + builder.addArtifacts(castList(params.get("files"), Artifact.class)); + } + if (params.containsKey("transitive_files")) { + builder.addTransitiveArtifacts(cast(params.get("transitive_files"), + SkylarkNestedSet.class, "files", loc).getSet(Artifact.class)); + } + return builder.build(); + } + }; + + @SkylarkBuiltin(name = "command_helper", doc = "Creates a command helper class.", + objectType = SkylarkRuleContext.class, + returnType = CommandHelper.class, + mandatoryParams = { + @Param(name = "tools", type = SkylarkList.class, generic1 = TransitiveInfoCollection.class, + doc = "list of tools (list of targets)"), + @Param(name = "label_dict", type = Map.class, + doc = "dictionary of resolved labels and the corresponding list of artifacts " + + "(a dict of Label : list of files)")}) + private static final SkylarkFunction createCommandHelper = + new SimpleSkylarkFunction("command_helper") { + @SuppressWarnings("unchecked") + @Override + protected Object call(Map<String, Object> params, Location loc) + throws ConversionException, EvalException { + SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self"); + return new CommandHelper(ctx.getRuleContext(), + AnalysisUtils.getProviders( + castList(params.get("tools"), TransitiveInfoCollection.class), + FilesToRunProvider.class), + // TODO(bazel-team): this cast to Map is unchecked and is not safe. + // The best way to fix this probably is to convert CommandHelper to Skylark. + ImmutableMap.copyOf((Map<Label, Iterable<Artifact>>) params.get("label_dict"))); + } + }; + + + @SkylarkBuiltin(name = "var", + doc = "get the value bound to a configuration variable in the context", + objectType = SkylarkRuleContext.class, + mandatoryParams = { + @Param(name = "name", type = String.class, doc = "the name of the variable") + }, + returnType = String.class) + private static final SkylarkFunction configurationMakeVariableContext = + new SimpleSkylarkFunction("var") { + @SuppressWarnings("unchecked") + @Override + protected Object call(Map<String, Object> params, Location loc) + throws ConversionException, EvalException { + SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self"); + String name = (String) params.get("name"); + try { + return ctx.getRuleContext().getConfigurationMakeVariableContext() + .lookupMakeVariable(name); + } catch (MakeVariableExpander.ExpansionException e) { + throw new EvalException(loc, "configuration variable " + + ShellEscaper.escapeString(name) + " not defined"); + } + } + }; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcBinary.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcBinary.java new file mode 100644 index 0000000000..6efcd9daeb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcBinary.java @@ -0,0 +1,635 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ParameterFile; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.Util; +import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.DynamicMode; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.rules.test.BaselineCoverageAction; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.util.OsUtils; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * A ConfiguredTarget for <code>cc_binary</code> rules. + */ +public abstract class CcBinary implements RuleConfiguredTargetFactory { + + private final CppSemantics semantics; + + protected CcBinary(CppSemantics semantics) { + this.semantics = semantics; + } + + // TODO(bazel-team): should this use Link.SHARED_LIBRARY_FILETYPES? + private static final FileTypeSet SHARED_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY); + + /** + * The maximum number of inputs for any single .dwp generating action. For cases where + * this value is exceeded, the action is split up into "batches" that fall under the limit. + * See {@link #createDebugPackagerActions} for details. + */ + @VisibleForTesting + public static final int MAX_INPUTS_PER_DWP_ACTION = 100; + + /** + * Intermediate dwps are written to this subdirectory under the main dwp's output path. + */ + @VisibleForTesting + public static final String INTERMEDIATE_DWP_DIR = "_dwps"; + + private static Runfiles collectRunfiles(RuleContext context, + CcCommon common, + CcLinkingOutputs linkingOutputs, + CppCompilationContext cppCompilationContext, + LinkStaticness linkStaticness, + NestedSet<Artifact> filesToBuild, + Iterable<Artifact> fakeLinkerInputs, + boolean fake) { + Runfiles.Builder builder = new Runfiles.Builder(); + Function<TransitiveInfoCollection, Runfiles> runfilesMapping = + CppRunfilesProvider.runfilesFunction(linkStaticness != LinkStaticness.DYNAMIC); + boolean linkshared = isLinkShared(context); + builder.addTransitiveArtifacts(filesToBuild); + // Add the shared libraries to the runfiles. This adds any shared libraries that are in the + // srcs of this target. + builder.addArtifacts(linkingOutputs.getLibrariesForRunfiles(true)); + builder.addRunfiles(context, RunfilesProvider.DEFAULT_RUNFILES); + builder.add(context, runfilesMapping); + CcToolchainProvider toolchain = CppHelper.getToolchain(context); + // Add the C++ runtime libraries if linking them dynamically. + if (linkStaticness == LinkStaticness.DYNAMIC) { + builder.addTransitiveArtifacts(toolchain.getDynamicRuntimeLinkInputs()); + } + // For cc_binary and cc_test rules, there is an implicit dependency on + // the malloc library package, which is specified by the "malloc" attribute. + // As the BUILD encyclopedia says, the "malloc" attribute should be ignored + // if linkshared=1. + if (!linkshared) { + TransitiveInfoCollection malloc = CppHelper.mallocForTarget(context); + builder.addTarget(malloc, RunfilesProvider.DEFAULT_RUNFILES); + builder.addTarget(malloc, runfilesMapping); + } + + if (fake) { + // Add the object files, libraries, and linker scripts that are used to + // link this executable. + builder.addSymlinksToArtifacts(Iterables.filter(fakeLinkerInputs, Artifact.MIDDLEMAN_FILTER)); + // The crosstool inputs for the link action are not sufficient; we also need the crosstool + // inputs for compilation. Node that these cannot be middlemen because Runfiles does not + // know how to expand them. + builder.addTransitiveArtifacts(toolchain.getCrosstool()); + builder.addTransitiveArtifacts(toolchain.getLibcLink()); + // Add the sources files that are used to compile the object files. + // We add the headers in the transitive closure and our own sources in the srcs + // attribute. We do not provide the auxiliary inputs, because they are only used when we + // do FDO compilation, and cc_fake_binary does not support FDO. + builder.addSymlinksToArtifacts( + Iterables.transform(common.getCAndCppSources(), Pair.<Artifact, Label>firstFunction())); + builder.addSymlinksToArtifacts(cppCompilationContext.getDeclaredIncludeSrcs()); + } + return builder.build(); + } + + @Override + public ConfiguredTarget create(RuleContext context) { + return CcBinary.init(semantics, context, /*fake =*/ false, /*useTestOnlyFlags =*/ false); + } + + public static ConfiguredTarget init(CppSemantics semantics, RuleContext ruleContext, boolean fake, + boolean useTestOnlyFlags) { + ruleContext.checkSrcsSamePackage(true); + CcCommon common = new CcCommon(ruleContext); + CppConfiguration cppConfiguration = ruleContext.getFragment(CppConfiguration.class); + + LinkTargetType linkType = + isLinkShared(ruleContext) ? LinkTargetType.DYNAMIC_LIBRARY : LinkTargetType.EXECUTABLE; + + CcLibraryHelper helper = new CcLibraryHelper(ruleContext, semantics) + .setLinkType(linkType) + .setHeadersCheckingMode(common.determineHeadersCheckingMode()) + .addCopts(common.getCopts()) + .setNoCopts(common.getNoCopts()) + .addLinkopts(common.getLinkopts()) + .addDefines(common.getDefines()) + .addCompilationPrerequisites(common.getSharedLibrariesFromSrcs()) + .addCompilationPrerequisites(common.getStaticLibrariesFromSrcs()) + .addSources(common.getCAndCppSources()) + .addPrivateHeaders(FileType.filter( + ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list(), + CppFileTypes.CPP_HEADER)) + .addObjectFiles(common.getObjectFilesFromSrcs(false)) + .addPicObjectFiles(common.getObjectFilesFromSrcs(true)) + .addPicIndependentObjectFiles(common.getLinkerScripts()) + .addDeps(ruleContext.getPrerequisites("deps", Mode.TARGET)) + .addDeps(ImmutableList.of(CppHelper.mallocForTarget(ruleContext))) + .setEnableLayeringCheck(ruleContext.getFeatures().contains(CppRuleClasses.LAYERING_CHECK)) + .addSystemIncludeDirs(common.getSystemIncludeDirs()) + .addIncludeDirs(common.getIncludeDirs()) + .addLooseIncludeDirs(common.getLooseIncludeDirs()) + .setFake(fake); + + CcLibraryHelper.Info info = helper.build(); + CppCompilationContext cppCompilationContext = info.getCppCompilationContext(); + CcCompilationOutputs ccCompilationOutputs = info.getCcCompilationOutputs(); + + // if cc_binary includes "linkshared=1", then gcc will be invoked with + // linkopt "-shared", which causes the result of linking to be a shared + // library. In this case, the name of the executable target should end + // in ".so". + PathFragment executableName = Util.getWorkspaceRelativePath( + ruleContext.getTarget(), "", OsUtils.executableExtension()); + CppLinkAction.Builder linkActionBuilder = determineLinkerArguments( + ruleContext, common, cppConfiguration, ccCompilationOutputs, + cppCompilationContext.getCompilationPrerequisites(), fake, executableName); + linkActionBuilder.setUseTestOnlyFlags(useTestOnlyFlags); + linkActionBuilder.addNonLibraryInputs(ccCompilationOutputs.getHeaderTokenFiles()); + + CcToolchainProvider ccToolchain = CppHelper.getToolchain(ruleContext); + LinkStaticness linkStaticness = getLinkStaticness(ruleContext, common, cppConfiguration); + if (linkStaticness == LinkStaticness.DYNAMIC) { + linkActionBuilder.setRuntimeInputs( + ccToolchain.getDynamicRuntimeLinkMiddleman(), + ccToolchain.getDynamicRuntimeLinkInputs()); + } else { + linkActionBuilder.setRuntimeInputs( + ccToolchain.getStaticRuntimeLinkMiddleman(), + ccToolchain.getStaticRuntimeLinkInputs()); + // Only force a static link of libgcc if static runtime linking is enabled (which + // can't be true if runtimeInputs is empty). + // TODO(bazel-team): Move this to CcToolchain. + if (!ccToolchain.getStaticRuntimeLinkInputs().isEmpty()) { + linkActionBuilder.addLinkopt("-static-libgcc"); + } + } + + linkActionBuilder.setLinkType(linkType); + linkActionBuilder.setLinkStaticness(linkStaticness); + linkActionBuilder.setFake(fake); + + // store immutable context now, recreate builder later + CppLinkAction.Context linkContext = new CppLinkAction.Context(linkActionBuilder); + + CppLinkAction linkAction = linkActionBuilder.build(); + ruleContext.registerAction(linkAction); + LibraryToLink outputLibrary = linkAction.getOutputLibrary(); + Iterable<Artifact> fakeLinkerInputs = + fake ? linkAction.getInputs() : ImmutableList.<Artifact>of(); + Artifact executable = outputLibrary.getArtifact(); + CcLinkingOutputs.Builder linkingOutputsBuilder = new CcLinkingOutputs.Builder(); + if (isLinkShared(ruleContext)) { + if (CppFileTypes.SHARED_LIBRARY.matches(executableName)) { + linkingOutputsBuilder.addDynamicLibrary(outputLibrary); + linkingOutputsBuilder.addExecutionDynamicLibrary(outputLibrary); + } else { + ruleContext.attributeError("linkshared", "'linkshared' used in non-shared library"); + } + } + // Also add all shared libraries from srcs. + for (Artifact library : common.getSharedLibrariesFromSrcs()) { + LibraryToLink symlink = common.getDynamicLibrarySymlink(library, true); + linkingOutputsBuilder.addDynamicLibrary(symlink); + linkingOutputsBuilder.addExecutionDynamicLibrary(symlink); + } + CcLinkingOutputs linkingOutputs = linkingOutputsBuilder.build(); + NestedSet<Artifact> filesToBuild = NestedSetBuilder.create(Order.STABLE_ORDER, executable); + + // Create the stripped binary, but don't add it to filesToBuild; it's only built when requested. + Artifact strippedFile = ruleContext.getImplicitOutputArtifact( + CppRuleClasses.CC_BINARY_STRIPPED); + createStripAction(ruleContext, cppConfiguration, executable, strippedFile); + + DwoArtifactsCollector dwoArtifacts = + collectTransitiveDwoArtifacts(ruleContext, common, cppConfiguration, ccCompilationOutputs); + Artifact dwpFile = + ruleContext.getImplicitOutputArtifact(CppRuleClasses.CC_BINARY_DEBUG_PACKAGE); + createDebugPackagerActions(ruleContext, cppConfiguration, dwpFile, dwoArtifacts); + + // The debug package should include the dwp file only if it was explicitly requested. + Artifact explicitDwpFile = dwpFile; + if (!cppConfiguration.useFission()) { + explicitDwpFile = null; + } + + // TODO(bazel-team): Do we need to put original shared libraries (along with + // mangled symlinks) into the RunfilesSupport object? It does not seem + // logical since all symlinked libraries will be linked anyway and would + // not require manual loading but if we do, then we would need to collect + // their names and use a different constructor below. + Runfiles runfiles = collectRunfiles(ruleContext, common, linkingOutputs, + cppCompilationContext, linkStaticness, filesToBuild, fakeLinkerInputs, fake); + RunfilesSupport runfilesSupport = RunfilesSupport.withExecutable( + ruleContext, runfiles, executable, ruleContext.getConfiguration().buildRunfiles()); + + TransitiveLipoInfoProvider transitiveLipoInfo; + if (cppConfiguration.isLipoContextCollector()) { + transitiveLipoInfo = common.collectTransitiveLipoLabels(ccCompilationOutputs); + } else { + transitiveLipoInfo = TransitiveLipoInfoProvider.EMPTY; + } + + RuleConfiguredTargetBuilder ruleBuilder = new RuleConfiguredTargetBuilder(ruleContext); + common.addTransitiveInfoProviders( + ruleBuilder, filesToBuild, ccCompilationOutputs, cppCompilationContext, linkingOutputs, + dwoArtifacts, transitiveLipoInfo); + + Map<Artifact, IncludeScannable> scannableMap = new LinkedHashMap<>(); + if (cppConfiguration.isLipoContextCollector()) { + for (IncludeScannable scannable : transitiveLipoInfo.getTransitiveIncludeScannables()) { + // These should all be CppCompileActions, which should have only one source file. + // This is also checked when they are put into the nested set. + Artifact source = + Iterables.getOnlyElement(scannable.getIncludeScannerSources()); + scannableMap.put(source, scannable); + } + } + + return ruleBuilder + .add(RunfilesProvider.class, RunfilesProvider.simple(runfiles)) + .add( + CppDebugPackageProvider.class, + new CppDebugPackageProvider(strippedFile, executable, explicitDwpFile)) + .setRunfilesSupport(runfilesSupport, executable) + .setBaselineCoverageArtifacts(createBaselineCoverageArtifacts( + ruleContext, common, ccCompilationOutputs, fake)) + .addProvider(LipoContextProvider.class, new LipoContextProvider( + cppCompilationContext, ImmutableMap.copyOf(scannableMap))) + .addProvider(CppLinkAction.Context.class, linkContext) + .build(); + } + + /** + * Creates an action to strip an executable. + */ + private static void createStripAction(RuleContext context, + CppConfiguration cppConfiguration, Artifact input, Artifact output) { + context.registerAction(new SpawnAction.Builder() + .addInput(input) + .addTransitiveInputs(CppHelper.getToolchain(context).getStrip()) + .addOutput(output) + .useDefaultShellEnvironment() + .setExecutable(cppConfiguration.getStripExecutable()) + .addArguments("-S", "-p", "-o", output.getExecPathString()) + .addArguments("-R", ".gnu.switches.text.quote_paths") + .addArguments("-R", ".gnu.switches.text.bracket_paths") + .addArguments("-R", ".gnu.switches.text.system_paths") + .addArguments("-R", ".gnu.switches.text.cpp_defines") + .addArguments("-R", ".gnu.switches.text.cpp_includes") + .addArguments("-R", ".gnu.switches.text.cl_args") + .addArguments("-R", ".gnu.switches.text.lipo_info") + .addArguments("-R", ".gnu.switches.text.annotation") + .addArguments(cppConfiguration.getStripOpts()) + .addArgument(input.getExecPathString()) + .setProgressMessage("Stripping " + output.prettyPrint() + " for " + context.getLabel()) + .setMnemonic("CcStrip") + .build(context)); + } + + /** + * Given 'temps', traverse this target and its dependencies and collect up all + * the object files, libraries, linker options, linkstamps attributes and linker scripts. + */ + private static CppLinkAction.Builder determineLinkerArguments(RuleContext context, + CcCommon common, CppConfiguration cppConfiguration, CcCompilationOutputs compilationOutputs, + ImmutableSet<Artifact> compilationPrerequisites, + boolean fake, PathFragment executableName) { + CppLinkAction.Builder builder = new CppLinkAction.Builder(context, executableName) + .setCrosstoolInputs(CppHelper.getToolchain(context).getLink()) + .addNonLibraryInputs(compilationPrerequisites); + + // Determine the object files to link in. + boolean usePic = CppHelper.usePic(context, !isLinkShared(context)) && !fake; + Iterable<Artifact> compiledObjectFiles = compilationOutputs.getObjectFiles(usePic); + + if (fake) { + builder.addFakeNonLibraryInputs(compiledObjectFiles); + } else { + builder.addNonLibraryInputs(compiledObjectFiles); + } + + builder.addNonLibraryInputs(common.getObjectFilesFromSrcs(usePic)); + builder.addNonLibraryInputs(common.getLinkerScripts()); + + // Determine the libraries to link in. + // First libraries from srcs. Shared library artifacts here are substituted with mangled symlink + // artifacts generated by getDynamicLibraryLink(). This is done to minimize number of -rpath + // entries during linking process. + for (Artifact library : common.getLibrariesFromSrcs()) { + if (SHARED_LIBRARY_FILETYPES.matches(library.getFilename())) { + builder.addLibrary(common.getDynamicLibrarySymlink(library, true)); + } else { + builder.addLibrary(LinkerInputs.opaqueLibraryToLink(library)); + } + } + + // Then libraries from the closure of deps. + List<String> linkopts = new ArrayList<>(); + Map<Artifact, ImmutableList<Artifact>> linkstamps = new LinkedHashMap<>(); + + NestedSet<LibraryToLink> librariesInDepsClosure = + findLibrariesToLinkInDepsClosure(context, common, cppConfiguration, linkopts, linkstamps); + builder.addLinkopts(linkopts); + builder.addLinkstamps(linkstamps); + + builder.addLibraries(librariesInDepsClosure); + return builder; + } + + /** + * Explore the transitive closure of our deps to collect linking information. + */ + private static NestedSet<LibraryToLink> findLibrariesToLinkInDepsClosure( + RuleContext context, + CcCommon common, + CppConfiguration cppConfiguration, + List<String> linkopts, + Map<Artifact, + ImmutableList<Artifact>> linkstamps) { + // This is true for both FULLY STATIC and MOSTLY STATIC linking. + boolean linkingStatically = + getLinkStaticness(context, common, cppConfiguration) != LinkStaticness.DYNAMIC; + + CcLinkParams linkParams = collectCcLinkParams( + context, common, linkingStatically, isLinkShared(context)); + linkopts.addAll(linkParams.flattenedLinkopts()); + linkstamps.putAll(CppHelper.resolveLinkstamps(context, linkParams)); + return linkParams.getLibraries(); + } + + /** + * Gets the linkopts to use for this binary. These options are NOT used when + * linking other binaries that depend on this binary. + * + * @return a new List instance that contains the linkopts for this binary + * target. + */ + private static ImmutableList<String> getBinaryLinkopts(RuleContext context, + CcCommon common) { + List<String> linkopts = new ArrayList<>(); + if (isLinkShared(context)) { + linkopts.add("-shared"); + } + linkopts.addAll(common.getLinkopts()); + return ImmutableList.copyOf(linkopts); + } + + private static boolean linkstaticAttribute(RuleContext context) { + return context.attributes().get("linkstatic", Type.BOOLEAN); + } + + /** + * Returns "true" if the {@code linkshared} attribute exists and is set. + */ + private static final boolean isLinkShared(RuleContext context) { + return context.getRule().getRuleClassObject().hasAttr("linkshared", Type.BOOLEAN) + && context.attributes().get("linkshared", Type.BOOLEAN); + } + + private static final boolean dashStaticInLinkopts(CcCommon common, + CppConfiguration cppConfiguration) { + return common.getLinkopts().contains("-static") + || cppConfiguration.getLinkOptions().contains("-static"); + } + + private static final LinkStaticness getLinkStaticness(RuleContext context, + CcCommon common, CppConfiguration cppConfiguration) { + if (cppConfiguration.getDynamicMode() == DynamicMode.FULLY) { + return LinkStaticness.DYNAMIC; + } else if (dashStaticInLinkopts(common, cppConfiguration)) { + return LinkStaticness.FULLY_STATIC; + } else if (cppConfiguration.getDynamicMode() == DynamicMode.OFF + || linkstaticAttribute(context)) { + return LinkStaticness.MOSTLY_STATIC; + } else { + return LinkStaticness.DYNAMIC; + } + } + + /** + * Collects .dwo artifacts either transitively or directly, depending on the link type. + * + * <p>For a cc_binary, we only include the .dwo files corresponding to the .o files that are + * passed into the link. For static linking, this includes all transitive dependencies. But + * for dynamic linking, dependencies are separately linked into their own shared libraries, + * so we don't need them here. + */ + private static DwoArtifactsCollector collectTransitiveDwoArtifacts(RuleContext context, + CcCommon common, CppConfiguration cppConfiguration, CcCompilationOutputs compilationOutputs) { + if (getLinkStaticness(context, common, cppConfiguration) == LinkStaticness.DYNAMIC) { + return DwoArtifactsCollector.directCollector(compilationOutputs); + } else { + return CcCommon.collectTransitiveDwoArtifacts(context, compilationOutputs); + } + } + + @VisibleForTesting + public static Iterable<Artifact> getDwpInputs( + RuleContext context, NestedSet<Artifact> picDwoArtifacts, NestedSet<Artifact> dwoArtifacts) { + return CppHelper.usePic(context, !isLinkShared(context)) ? picDwoArtifacts : dwoArtifacts; + } + + /** + * Creates the actions needed to generate this target's "debug info package" + * (i.e. its .dwp file). + */ + private static void createDebugPackagerActions(RuleContext context, + CppConfiguration cppConfiguration, Artifact dwpOutput, + DwoArtifactsCollector dwoArtifactsCollector) { + Iterable<Artifact> allInputs = getDwpInputs(context, + dwoArtifactsCollector.getPicDwoArtifacts(), + dwoArtifactsCollector.getDwoArtifacts()); + + // No inputs? Just generate a trivially empty .dwp. + // + // Note this condition automatically triggers for any build where fission is disabled. + // Because rules referencing .dwp targets may be invoked with or without fission, we need + // to support .dwp generation even when fission is disabled. Since no actual functionality + // is expected then, an empty file is appropriate. + if (Iterables.isEmpty(allInputs)) { + context.registerAction( + new FileWriteAction(context.getActionOwner(), dwpOutput, "", false)); + return; + } + + // Get the tool inputs necessary to run the dwp command. + NestedSet<Artifact> dwpTools = CppHelper.getToolchain(context).getDwp(); + Preconditions.checkState(!dwpTools.isEmpty()); + + // We apply a hierarchical action structure to limit the maximum number of inputs to any + // single action. + // + // While the dwp tools consumes .dwo files, it can also consume intermediate .dwp files, + // allowing us to split a large input set into smaller batches of arbitrary size and order. + // Aside from the parallelism performance benefits this offers, this also reduces input + // size requirements: if a.dwo, b.dwo, c.dwo, and e.dwo are each 1 KB files, we can apply + // two intermediate actions DWP(a.dwo, b.dwo) --> i1.dwp and DWP(c.dwo, e.dwo) --> i2.dwp. + // When we then apply the final action DWP(i1.dwp, i2.dwp) --> finalOutput.dwp, the inputs + // to this action will usually total far less than 4 KB. + // + // This list tracks every action we'll need to generate the output .dwp with batching. + List<SpawnAction.Builder> packagers = new ArrayList<>(); + + // Step 1: generate our batches. We currently break into arbitrary batches of fixed maximum + // input counts, but we can always apply more intelligent heuristics if the need arises. + SpawnAction.Builder currentPackager = newDwpAction(cppConfiguration, dwpTools); + int inputsForCurrentPackager = 0; + + for (Artifact dwoInput : allInputs) { + if (inputsForCurrentPackager == MAX_INPUTS_PER_DWP_ACTION) { + packagers.add(currentPackager); + currentPackager = newDwpAction(cppConfiguration, dwpTools); + inputsForCurrentPackager = 0; + } + currentPackager.addInputArgument(dwoInput); + inputsForCurrentPackager++; + } + packagers.add(currentPackager); + + // Step 2: given the batches, create the actions. + if (packagers.size() == 1) { + // If we only have one batch, make a single "original inputs --> final output" action. + context.registerAction(Iterables.getOnlyElement(packagers) + .addArgument("-o") + .addOutputArgument(dwpOutput) + .setMnemonic("CcGenerateDwp") + .build(context)); + } else { + // If we have multiple batches, make them all intermediate actions, then pipe their outputs + // into an additional action that outputs the final artifact. + // + // Note this only creates a hierarchy one level deep (i.e. we don't check if the number of + // intermediate outputs exceeds the maximum batch size). This is okay for current needs, + // which shouldn't stress those limits. + List<Artifact> intermediateOutputs = new ArrayList<>(); + + int count = 1; + for (SpawnAction.Builder packager : packagers) { + Artifact intermediateOutput = + getIntermediateDwpFile(context.getAnalysisEnvironment(), dwpOutput, count++); + context.registerAction(packager + .addArgument("-o") + .addOutputArgument(intermediateOutput) + .setMnemonic("CcGenerateIntermediateDwp") + .build(context)); + intermediateOutputs.add(intermediateOutput); + } + + // Now create the final action. + context.registerAction(newDwpAction(cppConfiguration, dwpTools) + .addInputArguments(intermediateOutputs) + .addArgument("-o") + .addOutputArgument(dwpOutput) + .setMnemonic("CcGenerateDwp") + .build(context)); + } + } + + /** + * Returns a new SpawnAction builder for generating dwp files, pre-initialized with + * standard settings. + */ + private static SpawnAction.Builder newDwpAction(CppConfiguration cppConfiguration, + NestedSet<Artifact> dwpTools) { + return new SpawnAction.Builder() + .addTransitiveInputs(dwpTools) + .setExecutable(cppConfiguration.getDwpExecutable()) + .useParameterFile(ParameterFile.ParameterFileType.UNQUOTED); + } + + /** + * Creates an intermediate dwp file keyed off the name and path of the final output. + */ + private static Artifact getIntermediateDwpFile(AnalysisEnvironment env, Artifact dwpOutput, + int orderNumber) { + PathFragment outputPath = dwpOutput.getRootRelativePath(); + PathFragment intermediatePath = + FileSystemUtils.appendWithoutExtension(outputPath, "-" + orderNumber); + return env.getDerivedArtifact( + outputPath.getParentDirectory().getRelative( + INTERMEDIATE_DWP_DIR + "/" + intermediatePath.getPathString()), + dwpOutput.getRoot()); + } + + /** + * Collect link parameters from the transitive closure. + */ + private static CcLinkParams collectCcLinkParams(RuleContext context, CcCommon common, + boolean linkingStatically, boolean linkShared) { + CcLinkParams.Builder builder = CcLinkParams.builder(linkingStatically, linkShared); + + if (isLinkShared(context)) { + // CcLinkingOutputs is empty because this target is not configured yet + builder.addCcLibrary(context, common, false, CcLinkingOutputs.EMPTY); + } else { + builder.addTransitiveTargets( + context.getPrerequisites("deps", Mode.TARGET), + CcLinkParamsProvider.TO_LINK_PARAMS, CcSpecificLinkParamsProvider.TO_LINK_PARAMS); + builder.addTransitiveTarget(CppHelper.mallocForTarget(context)); + builder.addLinkOpts(getBinaryLinkopts(context, common)); + } + return builder.build(); + } + + private static ImmutableList<Artifact> createBaselineCoverageArtifacts( + RuleContext context, CcCommon common, CcCompilationOutputs compilationOutputs, + boolean fake) { + if (!TargetUtils.isTestRule(context.getRule()) && !fake) { + Iterable<Artifact> objectFiles = compilationOutputs.getObjectFiles( + CppHelper.usePic(context, !isLinkShared(context))); + return BaselineCoverageAction.getBaselineCoverageArtifacts(context, + common.getInstrumentedFilesProvider(objectFiles).getInstrumentedFiles()); + } else { + return ImmutableList.of(); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java new file mode 100644 index 0000000000..3f5ff76a0d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java @@ -0,0 +1,678 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.CompilationPrerequisitesProvider; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToCompileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TempsProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.DynamicMode; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.HeadersCheckingMode; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.LocalMetadataCollector; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProviderImpl; +import com.google.devtools.build.lib.shell.ShellUtils; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * Common parts of the implementation of cc rules. + */ +public final class CcCommon { + + private static final String NO_COPTS_ATTRIBUTE = "nocopts"; + + private static final FileTypeSet SOURCE_TYPES = FileTypeSet.of( + CppFileTypes.CPP_SOURCE, + CppFileTypes.CPP_HEADER, + CppFileTypes.C_SOURCE, + CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR); + + /** + * Collects all metadata files generated by C++ compilation actions that output the .o files + * on the input. + */ + private static final LocalMetadataCollector CC_METADATA_COLLECTOR = + new LocalMetadataCollector() { + @Override + public void collectMetadataArtifacts(Iterable<Artifact> objectFiles, + AnalysisEnvironment analysisEnvironment, NestedSetBuilder<Artifact> metadataFilesBuilder) { + for (Artifact artifact : objectFiles) { + Action action = analysisEnvironment.getLocalGeneratingAction(artifact); + if (action instanceof CppCompileAction) { + addOutputs(metadataFilesBuilder, action, CppFileTypes.COVERAGE_NOTES); + } + } + } + }; + + /** C++ configuration */ + private final CppConfiguration cppConfiguration; + + /** The Artifacts from srcs. */ + private final ImmutableList<Artifact> sources; + + private final ImmutableList<Pair<Artifact, Label>> cAndCppSources; + + /** Expanded and tokenized copts attribute. Set by initCopts(). */ + private final ImmutableList<String> copts; + + /** + * The expanded linkopts for this rule. + */ + private final ImmutableList<String> linkopts; + + private final RuleContext ruleContext; + + public CcCommon(RuleContext ruleContext) { + this.ruleContext = ruleContext; + this.cppConfiguration = ruleContext.getFragment(CppConfiguration.class); + this.sources = hasAttribute("srcs", Type.LABEL_LIST) + ? ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list() + : ImmutableList.<Artifact>of(); + + this.cAndCppSources = collectCAndCppSources(); + copts = initCopts(); + linkopts = initLinkopts(); + } + + ImmutableList<Artifact> getTemps(CcCompilationOutputs compilationOutputs) { + return cppConfiguration.isLipoContextCollector() + ? ImmutableList.<Artifact>of() + : compilationOutputs.getTemps(); + } + + /** + * Returns our own linkopts from the rule attribute. This determines linker + * options to use when building this target and anything that depends on it. + */ + public ImmutableList<String> getLinkopts() { + return linkopts; + } + + public ImmutableList<String> getCopts() { + return copts; + } + + private boolean hasAttribute(String name, Type<?> type) { + return ruleContext.getRule().getRuleClassObject().hasAttr(name, type); + } + + private static NestedSet<Artifact> collectExecutionDynamicLibraryArtifacts( + RuleContext ruleContext, + List<LibraryToLink> executionDynamicLibraries) { + Iterable<Artifact> artifacts = LinkerInputs.toLibraryArtifacts(executionDynamicLibraries); + if (!Iterables.isEmpty(artifacts)) { + return NestedSetBuilder.wrap(Order.STABLE_ORDER, artifacts); + } + + Iterable<CcExecutionDynamicLibrariesProvider> deps = ruleContext + .getPrerequisites("deps", Mode.TARGET, CcExecutionDynamicLibrariesProvider.class); + + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (CcExecutionDynamicLibrariesProvider dep : deps) { + builder.addTransitive(dep.getExecutionDynamicLibraryArtifacts()); + } + return builder.build(); + } + + /** + * Collects all .dwo artifacts in this target's transitive closure. + */ + public static DwoArtifactsCollector collectTransitiveDwoArtifacts( + RuleContext ruleContext, + CcCompilationOutputs compilationOutputs) { + ImmutableList.Builder<TransitiveInfoCollection> deps = + ImmutableList.<TransitiveInfoCollection>builder(); + + deps.addAll(ruleContext.getPrerequisites("deps", Mode.TARGET)); + + if (ruleContext.getRule().getRuleClassObject().hasAttr("malloc", Type.LABEL)) { + deps.add(CppHelper.mallocForTarget(ruleContext)); + } + if (ruleContext.getRule().getRuleClassObject().hasAttr("implementation", Type.LABEL_LIST)) { + deps.addAll(ruleContext.getPrerequisites("implementation", Mode.TARGET)); + } + + return compilationOutputs == null // Possible in LIPO collection mode (see initializationHook). + ? DwoArtifactsCollector.emptyCollector() + : DwoArtifactsCollector.transitiveCollector(compilationOutputs, deps.build()); + } + + public TransitiveLipoInfoProvider collectTransitiveLipoLabels(CcCompilationOutputs outputs) { + if (cppConfiguration.getFdoSupport().getFdoRoot() == null + || !cppConfiguration.isLipoContextCollector()) { + return TransitiveLipoInfoProvider.EMPTY; + } + + NestedSetBuilder<IncludeScannable> scannableBuilder = NestedSetBuilder.stableOrder(); + CppHelper.addTransitiveLipoInfoForCommonAttributes(ruleContext, outputs, scannableBuilder); + if (hasAttribute("implementation", Type.LABEL_LIST)) { + for (TransitiveLipoInfoProvider impl : AnalysisUtils.getProviders( + ruleContext.getPrerequisites("implementation", Mode.TARGET), + TransitiveLipoInfoProvider.class)) { + scannableBuilder.addTransitive(impl.getTransitiveIncludeScannables()); + } + } + + return new TransitiveLipoInfoProvider(scannableBuilder.build()); + } + + private NestedSet<LinkerInput> collectTransitiveCcNativeLibraries( + RuleContext ruleContext, + List<? extends LinkerInput> dynamicLibraries) { + NestedSetBuilder<LinkerInput> builder = NestedSetBuilder.linkOrder(); + builder.addAll(dynamicLibraries); + for (CcNativeLibraryProvider dep : + ruleContext.getPrerequisites("deps", Mode.TARGET, CcNativeLibraryProvider.class)) { + builder.addTransitive(dep.getTransitiveCcNativeLibraries()); + } + return builder.build(); + } + + /** + * Returns a list of ({@link Artifact}, {@link Label}) pairs. Each pair represents an input + * source file and the label of the rule that generates it (or the label of the source file + * itself if it is an input file) + */ + ImmutableList<Pair<Artifact, Label>> getCAndCppSources() { + return cAndCppSources; + } + + private boolean shouldProcessHeaders() { + boolean crosstoolSupportsHeaderParsing = + CppHelper.getToolchain(ruleContext).supportsHeaderParsing(); + return crosstoolSupportsHeaderParsing && ( + ruleContext.getFeatures().contains(CppRuleClasses.PREPROCESS_HEADERS) + || ruleContext.getFeatures().contains(CppRuleClasses.PARSE_HEADERS)); + } + + private ImmutableList<Pair<Artifact, Label>> collectCAndCppSources() { + Map<Artifact, Label> map = Maps.newLinkedHashMap(); + if (!hasAttribute("srcs", Type.LABEL_LIST)) { + return ImmutableList.<Pair<Artifact, Label>>of(); + } + Iterable<FileProvider> providers = + ruleContext.getPrerequisites("srcs", Mode.TARGET, FileProvider.class); + // TODO(bazel-team): Move header processing logic down in the stack (to CcLibraryHelper or + // such). + boolean processHeaders = shouldProcessHeaders(); + if (processHeaders && hasAttribute("hdrs", Type.LABEL_LIST)) { + providers = Iterables.concat(providers, + ruleContext.getPrerequisites("hdrs", Mode.TARGET, FileProvider.class)); + } + for (FileProvider provider : providers) { + for (Artifact artifact : FileType.filter(provider.getFilesToBuild(), SOURCE_TYPES)) { + boolean isHeader = CppFileTypes.CPP_HEADER.matches(artifact.getExecPath()); + if ((isHeader && !processHeaders) + || CppFileTypes.CPP_TEXTUAL_INCLUDE.matches(artifact.getExecPath())) { + continue; + } + Label oldLabel = map.put(artifact, provider.getLabel()); + // TODO(bazel-team): We currently do not warn for duplicate headers with + // different labels, as that would require cleaning up the code base + // without significant benefit; we should eventually make this + // consistent one way or the other. + if (!isHeader && oldLabel != null && !oldLabel.equals(provider.getLabel())) { + ruleContext.attributeError("srcs", String.format( + "Artifact '%s' is duplicated (through '%s' and '%s')", + artifact.getExecPathString(), oldLabel, provider.getLabel())); + } + } + } + + ImmutableList.Builder<Pair<Artifact, Label>> result = ImmutableList.builder(); + for (Map.Entry<Artifact, Label> entry : map.entrySet()) { + result.add(Pair.of(entry.getKey(), entry.getValue())); + } + + return result.build(); + } + + Iterable<Artifact> getLibrariesFromSrcs() { + return FileType.filter(sources, CppFileTypes.ARCHIVE, CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_LIBRARY, CppFileTypes.ALWAYS_LINK_PIC_LIBRARY, + CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY); + } + + Iterable<Artifact> getSharedLibrariesFromSrcs() { + return getSharedLibrariesFrom(sources); + } + + static Iterable<Artifact> getSharedLibrariesFrom(Iterable<Artifact> collection) { + return FileType.filter(collection, CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY); + } + + Iterable<Artifact> getStaticLibrariesFromSrcs() { + return FileType.filter(sources, CppFileTypes.ARCHIVE, CppFileTypes.ALWAYS_LINK_LIBRARY); + } + + Iterable<LibraryToLink> getPicStaticLibrariesFromSrcs() { + return LinkerInputs.opaqueLibrariesToLink( + FileType.filter(sources, CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_PIC_LIBRARY)); + } + + Iterable<Artifact> getObjectFilesFromSrcs(final boolean usePic) { + if (usePic) { + return Iterables.filter(sources, new Predicate<Artifact>() { + @Override + public boolean apply(Artifact artifact) { + String filename = artifact.getExecPathString(); + + // For compatibility with existing BUILD files, any ".o" files listed + // in srcs are assumed to be position-independent code, or + // at least suitable for inclusion in shared libraries, unless they + // end with ".nopic.o". (The ".nopic.o" extension is an undocumented + // feature to give users at least some control over this.) Note that + // some target platforms do not require shared library code to be PIC. + return CppFileTypes.PIC_OBJECT_FILE.matches(filename) || + (CppFileTypes.OBJECT_FILE.matches(filename) && !filename.endsWith(".nopic.o")); + } + }); + } else { + return FileType.filter(sources, CppFileTypes.OBJECT_FILE); + } + } + + /** + * Returns the files from headers and does some sanity checks. Note that this method reports + * warnings to the {@link RuleContext} as a side effect, and so should only be called once for any + * given rule. + */ + public static List<Artifact> getHeaders(RuleContext ruleContext) { + List<Artifact> hdrs = new ArrayList<>(); + for (TransitiveInfoCollection target : + ruleContext.getPrerequisitesIf("hdrs", Mode.TARGET, FileProvider.class)) { + FileProvider provider = target.getProvider(FileProvider.class); + for (Artifact artifact : provider.getFilesToBuild()) { + if (!CppRuleClasses.DISALLOWED_HDRS_FILES.matches(artifact.getFilename())) { + hdrs.add(artifact); + } else { + ruleContext.attributeWarning("hdrs", "file '" + artifact.getFilename() + + "' from target '" + target.getLabel() + "' is not allowed in hdrs"); + } + } + } + return hdrs; + } + + /** + * Uses {@link #getHeaders(RuleContext)} to get the {@code hdrs} on this target. This method will + * return an empty list if there is no {@code hdrs} attribute on this rule type. + */ + List<Artifact> getHeaders() { + if (!hasAttribute("hdrs", Type.LABEL_LIST)) { + return ImmutableList.of(); + } + return getHeaders(ruleContext); + } + + HeadersCheckingMode determineHeadersCheckingMode() { + HeadersCheckingMode headersCheckingMode = cppConfiguration.getHeadersCheckingMode(); + + // Package default overrides command line option. + if (ruleContext.getRule().getPackage().isDefaultHdrsCheckSet()) { + String value = + ruleContext.getRule().getPackage().getDefaultHdrsCheck().toUpperCase(Locale.ENGLISH); + headersCheckingMode = HeadersCheckingMode.valueOf(value); + } + + // 'hdrs_check' attribute overrides package default. + if (hasAttribute("hdrs_check", Type.STRING) + && ruleContext.getRule().isAttributeValueExplicitlySpecified("hdrs_check")) { + try { + String value = ruleContext.attributes().get("hdrs_check", Type.STRING) + .toUpperCase(Locale.ENGLISH); + headersCheckingMode = HeadersCheckingMode.valueOf(value); + } catch (IllegalArgumentException e) { + ruleContext.attributeError("hdrs_check", "must be one of: 'loose', 'warn' or 'strict'"); + } + } + + return headersCheckingMode; + } + + /** + * Expand and tokenize the copts and nocopts attributes. + */ + private ImmutableList<String> initCopts() { + if (!hasAttribute("copts", Type.STRING_LIST)) { + return ImmutableList.<String>of(); + } + // TODO(bazel-team): getAttributeCopts should not tokenize the strings. + // Make a warning for now. + List<String> tokens = new ArrayList<>(); + for (String str : ruleContext.attributes().get("copts", Type.STRING_LIST)) { + tokens.clear(); + try { + ShellUtils.tokenize(tokens, str); + if (tokens.size() > 1) { + ruleContext.attributeWarning("copts", + "each item in the list should contain only one option"); + } + } catch (ShellUtils.TokenizationException e) { + // ignore, the error is reported in the getAttributeCopts call + } + } + + Pattern nocopts = getNoCopts(ruleContext); + if (nocopts != null && nocopts.matcher("-Wno-future-warnings").matches()) { + ruleContext.attributeWarning("nocopts", + "Regular expression '" + nocopts.pattern() + "' is too general; for example, it matches " + + "'-Wno-future-warnings'. Thus it might *re-enable* compiler warnings we wish to " + + "disable globally. To disable all compiler warnings, add '-w' to copts instead"); + } + + return ImmutableList.<String>builder() + .addAll(getPackageCopts(ruleContext)) + .addAll(CppHelper.getAttributeCopts(ruleContext, "copts")) + .build(); + } + + private static ImmutableList<String> getPackageCopts(RuleContext ruleContext) { + List<String> unexpanded = ruleContext.getRule().getPackage().getDefaultCopts(); + return ImmutableList.copyOf(CppHelper.expandMakeVariables(ruleContext, "copts", unexpanded)); + } + + Pattern getNoCopts() { + return getNoCopts(ruleContext); + } + + /** + * Returns nocopts pattern built from the make variable expanded nocopts + * attribute. + */ + private static Pattern getNoCopts(RuleContext ruleContext) { + Pattern nocopts = null; + if (ruleContext.getRule().isAttrDefined(NO_COPTS_ATTRIBUTE, Type.STRING)) { + String nocoptsAttr = ruleContext.expandMakeVariables(NO_COPTS_ATTRIBUTE, + ruleContext.attributes().get(NO_COPTS_ATTRIBUTE, Type.STRING)); + try { + nocopts = Pattern.compile(nocoptsAttr); + } catch (PatternSyntaxException e) { + ruleContext.attributeError(NO_COPTS_ATTRIBUTE, + "invalid regular expression '" + nocoptsAttr + "': " + e.getMessage()); + } + } + return nocopts; + } + + // TODO(bazel-team): calculating nocopts every time is not very efficient, + // fix this after the rule migration. The problem is that in some cases we call this after + // the RCT is created (so RuleContext is not accessible), in some cases during the creation. + // It would probably make more sense to use TransitiveInfoProviders. + /** + * Returns true if the rule context has a nocopts regex that matches the given value, false + * otherwise. + */ + static boolean noCoptsMatches(String option, RuleContext ruleContext) { + Pattern nocopts = getNoCopts(ruleContext); + return nocopts == null ? false : nocopts.matcher(option).matches(); + } + + private static final String DEFINES_ATTRIBUTE = "defines"; + + /** + * Returns a list of define tokens from "defines" attribute. + * + * <p>We tokenize the "defines" attribute, to ensure that the handling of + * quotes and backslash escapes is consistent Bazel's treatment of the "copts" attribute. + * + * <p>But we require that the "defines" attribute consists of a single token. + */ + public List<String> getDefines() { + List<String> defines = new ArrayList<>(); + for (String define : + ruleContext.attributes().get(DEFINES_ATTRIBUTE, Type.STRING_LIST)) { + List<String> tokens = new ArrayList<>(); + try { + ShellUtils.tokenize(tokens, ruleContext.expandMakeVariables(DEFINES_ATTRIBUTE, define)); + if (tokens.size() == 1) { + defines.add(tokens.get(0)); + } else if (tokens.isEmpty()) { + ruleContext.attributeError(DEFINES_ATTRIBUTE, "empty definition not allowed"); + } else { + ruleContext.attributeError(DEFINES_ATTRIBUTE, + "definition contains too many tokens (found " + tokens.size() + + ", expecting exactly one)"); + } + } catch (ShellUtils.TokenizationException e) { + ruleContext.attributeError(DEFINES_ATTRIBUTE, e.getMessage()); + } + } + return defines; + } + + /** + * Collects our own linkopts from the rule attribute. This determines linker + * options to use when building this library and anything that depends on it. + */ + private final ImmutableList<String> initLinkopts() { + if (!hasAttribute("linkopts", Type.STRING_LIST)) { + return ImmutableList.<String>of(); + } + List<String> ourLinkopts = ruleContext.attributes().get("linkopts", Type.STRING_LIST); + List<String> result = new ArrayList<>(); + if (ourLinkopts != null) { + boolean allowDashStatic = !cppConfiguration.forceIgnoreDashStatic() + && (cppConfiguration.getDynamicMode() != DynamicMode.FULLY); + for (String linkopt : ourLinkopts) { + if (linkopt.equals("-static") && !allowDashStatic) { + continue; + } + CppHelper.expandAttribute(ruleContext, result, "linkopts", linkopt, true); + } + } + return ImmutableList.copyOf(result); + } + + /** + * Determines a list of loose include directories that are only allowed to be referenced when + * headers checking is {@link HeadersCheckingMode#LOOSE} or {@link HeadersCheckingMode#WARN}. + */ + List<PathFragment> getLooseIncludeDirs() { + List<PathFragment> result = new ArrayList<>(); + // The package directory of the rule contributes includes. Note that this also covers all + // non-subpackage sub-directories. + PathFragment rulePackage = ruleContext.getLabel().getPackageFragment(); + result.add(rulePackage); + + // Gather up all the dirs from the rule's srcs as well as any of the srcs outputs. + if (hasAttribute("srcs", Type.LABEL_LIST)) { + for (FileProvider src : + ruleContext.getPrerequisites("srcs", Mode.TARGET, FileProvider.class)) { + PathFragment packageDir = src.getLabel().getPackageFragment(); + for (Artifact a : src.getFilesToBuild()) { + result.add(packageDir); + // Attempt to gather subdirectories that might contain include files. + result.add(a.getRootRelativePath().getParentDirectory()); + } + } + } + + // Add in any 'includes' attribute values as relative path fragments + if (ruleContext.getRule().isAttributeValueExplicitlySpecified("includes")) { + PathFragment packageFragment = ruleContext.getLabel().getPackageFragment(); + // For now, anything with an 'includes' needs a blanket declaration + result.add(packageFragment.getRelative("**")); + } + return result; + } + + List<PathFragment> getSystemIncludeDirs() { + // Add in any 'includes' attribute values as relative path fragments + if (!ruleContext.getRule().isAttributeValueExplicitlySpecified("includes") + || !cppConfiguration.useIsystemForIncludes()) { + return ImmutableList.of(); + } + return getIncludeDirsFromIncludesAttribute(); + } + + List<PathFragment> getIncludeDirs() { + if (!ruleContext.getRule().isAttributeValueExplicitlySpecified("includes") + || cppConfiguration.useIsystemForIncludes()) { + return ImmutableList.of(); + } + return getIncludeDirsFromIncludesAttribute(); + } + + private List<PathFragment> getIncludeDirsFromIncludesAttribute() { + List<PathFragment> result = new ArrayList<>(); + PathFragment packageFragment = ruleContext.getLabel().getPackageFragment(); + for (String includesAttr : ruleContext.attributes().get("includes", Type.STRING_LIST)) { + includesAttr = ruleContext.expandMakeVariables("includes", includesAttr); + if (includesAttr.startsWith("/")) { + ruleContext.attributeWarning("includes", + "ignoring invalid absolute path '" + includesAttr + "'"); + continue; + } + PathFragment includesPath = packageFragment.getRelative(includesAttr).normalize(); + if (!includesPath.isNormalized()) { + ruleContext.attributeError("includes", + "Path references a path above the execution root."); + } + result.add(includesPath); + result.add(ruleContext.getConfiguration().getGenfilesFragment().getRelative(includesPath)); + } + return result; + } + + /** + * Collects compilation prerequisite artifacts. + */ + static CompilationPrerequisitesProvider collectCompilationPrerequisites( + RuleContext ruleContext, CppCompilationContext context) { + // TODO(bazel-team): Use context.getCompilationPrerequisites() instead. + NestedSetBuilder<Artifact> prerequisites = NestedSetBuilder.stableOrder(); + if (ruleContext.getRule().getRuleClassObject().hasAttr("srcs", Type.LABEL_LIST)) { + for (FileProvider provider : ruleContext + .getPrerequisites("srcs", Mode.TARGET, FileProvider.class)) { + prerequisites.addAll(FileType.filter(provider.getFilesToBuild(), SOURCE_TYPES)); + } + } + prerequisites.addTransitive(context.getDeclaredIncludeSrcs()); + return new CompilationPrerequisitesProvider(prerequisites.build()); + } + + /** + * Replaces shared library artifact with mangled symlink and creates related + * symlink action. For artifacts that should retain filename (e.g. libraries + * with SONAME tag), link is created to the parent directory instead. + * + * This action is performed to minimize number of -rpath entries used during + * linking process (by essentially "collecting" as many shared libraries as + * possible in the single directory), since we will be paying quadratic price + * for each additional entry on the -rpath. + * + * @param library Shared library artifact that needs to be mangled + * @param preserveName true if filename should be preserved, false - mangled. + * @return mangled symlink artifact. + */ + public LibraryToLink getDynamicLibrarySymlink(Artifact library, boolean preserveName) { + return SolibSymlinkAction.getDynamicLibrarySymlink( + ruleContext, library, preserveName, true, ruleContext.getConfiguration()); + } + + /** + * Returns any linker scripts found in the dependencies of the rule. + */ + Iterable<Artifact> getLinkerScripts() { + return FileType.filter(ruleContext.getPrerequisiteArtifacts("deps", Mode.TARGET).list(), + CppFileTypes.LINKER_SCRIPT); + } + + ImmutableList<Artifact> getFilesToCompile(CcCompilationOutputs compilationOutputs) { + return cppConfiguration.isLipoContextCollector() + ? ImmutableList.<Artifact>of() + : compilationOutputs.getObjectFiles(CppHelper.usePic(ruleContext, false)); + } + + InstrumentedFilesProvider getInstrumentedFilesProvider(Iterable<Artifact> files) { + return cppConfiguration.isLipoContextCollector() + ? InstrumentedFilesProviderImpl.EMPTY + : new InstrumentedFilesProviderImpl(new InstrumentedFilesCollector( + ruleContext, CppRuleClasses.INSTRUMENTATION_SPEC, CC_METADATA_COLLECTOR, files)); + } + + public static FeatureConfiguration configureFeatures(RuleContext ruleContext) { + CcToolchainProvider toolchain = CppHelper.getToolchain(ruleContext); + Set<String> requestedFeatures = ImmutableSet.of(CppRuleClasses.MODULE_MAP_HOME_CWD); + return toolchain.getFeatures().getFeatureConfiguration(requestedFeatures); + } + + public void addTransitiveInfoProviders(RuleConfiguredTargetBuilder builder, + NestedSet<Artifact> filesToBuild, + CcCompilationOutputs ccCompilationOutputs, + CppCompilationContext cppCompilationContext, + CcLinkingOutputs linkingOutputs, + DwoArtifactsCollector dwoArtifacts, + TransitiveLipoInfoProvider transitiveLipoInfo) { + List<Artifact> instrumentedObjectFiles = new ArrayList<>(); + instrumentedObjectFiles.addAll(ccCompilationOutputs.getObjectFiles(false)); + instrumentedObjectFiles.addAll(ccCompilationOutputs.getObjectFiles(true)); + builder + .setFilesToBuild(filesToBuild) + .add(CppCompilationContext.class, cppCompilationContext) + .add(TransitiveLipoInfoProvider.class, transitiveLipoInfo) + .add(CcExecutionDynamicLibrariesProvider.class, + new CcExecutionDynamicLibrariesProvider(collectExecutionDynamicLibraryArtifacts( + ruleContext, linkingOutputs.getExecutionDynamicLibraries()))) + .add(CcNativeLibraryProvider.class, new CcNativeLibraryProvider( + collectTransitiveCcNativeLibraries(ruleContext, linkingOutputs.getDynamicLibraries()))) + .add(InstrumentedFilesProvider.class, getInstrumentedFilesProvider( + instrumentedObjectFiles)) + .add(FilesToCompileProvider.class, new FilesToCompileProvider( + getFilesToCompile(ccCompilationOutputs))) + .add(CompilationPrerequisitesProvider.class, + collectCompilationPrerequisites(ruleContext, cppCompilationContext)) + .add(TempsProvider.class, new TempsProvider(getTemps(ccCompilationOutputs))) + .add(CppDebugFileProvider.class, new CppDebugFileProvider( + dwoArtifacts.getDwoArtifacts(), + dwoArtifacts.getPicDwoArtifacts())); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java new file mode 100644 index 0000000000..b9fa4e8f49 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java @@ -0,0 +1,207 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * A structured representation of the compilation outputs of a C++ rule. + */ +public class CcCompilationOutputs { + /** + * All .o files built by the target. + */ + private final ImmutableList<Artifact> objectFiles; + + /** + * All .pic.o files built by the target. + */ + private final ImmutableList<Artifact> picObjectFiles; + + /** + * All .dwo files built by the target, corresponding to .o outputs. + */ + private final ImmutableList<Artifact> dwoFiles; + + /** + * All .pic.dwo files built by the target, corresponding to .pic.o outputs. + */ + private final ImmutableList<Artifact> picDwoFiles; + + /** + * All artifacts that are created if "--save_temps" is true. + */ + private final ImmutableList<Artifact> temps; + + /** + * All token .h.processed files created when preprocessing or parsing headers. + */ + private final ImmutableList<Artifact> headerTokenFiles; + + private final List<IncludeScannable> lipoScannables; + + private CcCompilationOutputs(ImmutableList<Artifact> objectFiles, + ImmutableList<Artifact> picObjectFiles, ImmutableList<Artifact> dwoFiles, + ImmutableList<Artifact> picDwoFiles, ImmutableList<Artifact> temps, + ImmutableList<Artifact> headerTokenFiles, + ImmutableList<IncludeScannable> lipoScannables) { + this.objectFiles = objectFiles; + this.picObjectFiles = picObjectFiles; + this.dwoFiles = dwoFiles; + this.picDwoFiles = picDwoFiles; + this.temps = temps; + this.headerTokenFiles = headerTokenFiles; + this.lipoScannables = lipoScannables; + } + + /** + * Returns an unmodifiable view of the .o or .pic.o files set. + * + * @param usePic whether to return .pic.o files + */ + public ImmutableList<Artifact> getObjectFiles(boolean usePic) { + return usePic ? picObjectFiles : objectFiles; + } + + /** + * Returns an unmodifiable view of the .dwo files set. + */ + public ImmutableList<Artifact> getDwoFiles() { + return dwoFiles; + } + + /** + * Returns an unmodifiable view of the .pic.dwo files set. + */ + public ImmutableList<Artifact> getPicDwoFiles() { + return picDwoFiles; + } + + /** + * Returns an unmodifiable view of the temp files set. + */ + public ImmutableList<Artifact> getTemps() { + return temps; + } + + /** + * Returns an unmodifiable view of the .h.processed files. + */ + public Iterable<Artifact> getHeaderTokenFiles() { + return headerTokenFiles; + } + + /** + * Returns the {@link IncludeScannable} objects this C++ compile action contributes to a + * LIPO context collector. + */ + public List<IncludeScannable> getLipoScannables() { + return lipoScannables; + } + + public static final class Builder { + private final Set<Artifact> objectFiles = new LinkedHashSet<>(); + private final Set<Artifact> picObjectFiles = new LinkedHashSet<>(); + private final Set<Artifact> dwoFiles = new LinkedHashSet<>(); + private final Set<Artifact> picDwoFiles = new LinkedHashSet<>(); + private final Set<Artifact> temps = new LinkedHashSet<>(); + private final Set<Artifact> headerTokenFiles = new LinkedHashSet<>(); + private final List<IncludeScannable> lipoScannables = new ArrayList<>(); + + public CcCompilationOutputs build() { + return new CcCompilationOutputs(ImmutableList.copyOf(objectFiles), + ImmutableList.copyOf(picObjectFiles), ImmutableList.copyOf(dwoFiles), + ImmutableList.copyOf(picDwoFiles), ImmutableList.copyOf(temps), + ImmutableList.copyOf(headerTokenFiles), + ImmutableList.copyOf(lipoScannables)); + } + + public Builder merge(CcCompilationOutputs outputs) { + this.objectFiles.addAll(outputs.objectFiles); + this.picObjectFiles.addAll(outputs.picObjectFiles); + this.dwoFiles.addAll(outputs.dwoFiles); + this.picDwoFiles.addAll(outputs.picDwoFiles); + this.temps.addAll(outputs.temps); + this.headerTokenFiles.addAll(outputs.headerTokenFiles); + this.lipoScannables.addAll(outputs.lipoScannables); + return this; + } + + /** + * Adds an .o file. + */ + public Builder addObjectFile(Artifact artifact) { + objectFiles.add(artifact); + return this; + } + + public Builder addObjectFiles(Iterable<Artifact> artifacts) { + Iterables.addAll(objectFiles, artifacts); + return this; + } + + /** + * Adds a .pic.o file. + */ + public Builder addPicObjectFile(Artifact artifact) { + picObjectFiles.add(artifact); + return this; + } + + public Builder addPicObjectFiles(Iterable<Artifact> artifacts) { + Iterables.addAll(picObjectFiles, artifacts); + return this; + } + + public Builder addDwoFile(Artifact artifact) { + dwoFiles.add(artifact); + return this; + } + + public Builder addPicDwoFile(Artifact artifact) { + picDwoFiles.add(artifact); + return this; + } + + /** + * Adds temp files. + */ + public Builder addTemps(Iterable<Artifact> artifacts) { + Iterables.addAll(temps, artifacts); + return this; + } + + public Builder addHeaderTokenFile(Artifact artifact) { + headerTokenFiles.add(artifact); + return this; + } + + /** + * Adds an {@link IncludeScannable} that this compilation output object contributes to a + * LIPO context collector. + */ + public Builder addLipoScannable(IncludeScannable scannable) { + lipoScannables.add(scannable); + return this; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcExecutionDynamicLibrariesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcExecutionDynamicLibrariesProvider.java new file mode 100644 index 0000000000..39ce942f60 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcExecutionDynamicLibrariesProvider.java @@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A target that provides the execution-time dynamic libraries of a C++ rule. + */ +@Immutable +public final class CcExecutionDynamicLibrariesProvider implements TransitiveInfoProvider { + public static final CcExecutionDynamicLibrariesProvider EMPTY = + new CcExecutionDynamicLibrariesProvider( + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER)); + + private final NestedSet<Artifact> ccExecutionDynamicLibraries; + + public CcExecutionDynamicLibrariesProvider(NestedSet<Artifact> ccExecutionDynamicLibraries) { + this.ccExecutionDynamicLibraries = ccExecutionDynamicLibraries; + } + + /** + * Returns the execution-time dynamic libraries. + * + * <p>This normally returns the dynamic library created by the rule itself. However, if the rule + * does not create any dynamic libraries, then it returns the combined results of calling + * getExecutionDynamicLibraryArtifacts on all the rule's deps. This behaviour is so that this + * method is useful for a cc_library with deps but no srcs. + */ + public NestedSet<Artifact> getExecutionDynamicLibraryArtifacts() { + return ccExecutionDynamicLibraries; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibrary.java new file mode 100644 index 0000000000..428eedb6e2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibrary.java @@ -0,0 +1,395 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AlwaysBuiltArtifactsProvider; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.rules.test.BaselineCoverageAction; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.List; + +/** + * A ConfiguredTarget for <code>cc_library</code> rules. + */ +public abstract class CcLibrary implements RuleConfiguredTargetFactory { + + private final CppSemantics semantics; + + protected CcLibrary(CppSemantics semantics) { + this.semantics = semantics; + } + + // These file extensions don't generate object files. + private static final FileTypeSet NO_OBJECT_GENERATING_FILETYPES = FileTypeSet.of( + CppFileTypes.CPP_HEADER, CppFileTypes.ARCHIVE, CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_LIBRARY, CppFileTypes.ALWAYS_LINK_PIC_LIBRARY, + CppFileTypes.SHARED_LIBRARY); + + private static final Predicate<LibraryToLink> PIC_STATIC_FILTER = new Predicate<LibraryToLink>() { + @Override + public boolean apply(LibraryToLink input) { + String name = input.getArtifact().getExecPath().getBaseName(); + return !name.endsWith(".nopic.a") && !name.endsWith(".nopic.lo"); + } + }; + + private static Runfiles collectRunfiles(RuleContext context, + CcLinkingOutputs ccLinkingOutputs, + boolean neverLink, boolean addDynamicRuntimeInputArtifactsToRunfiles, + boolean linkingStatically) { + Runfiles.Builder builder = new Runfiles.Builder(); + + // neverlink= true creates a library that will never be linked into any binary that depends on + // it, but instead be loaded as an extension. So we need the dynamic library for this in the + // runfiles. + builder.addArtifacts(ccLinkingOutputs.getLibrariesForRunfiles(linkingStatically && !neverLink)); + builder.add(context, CppRunfilesProvider.runfilesFunction(linkingStatically)); + if (context.getRule().isAttrDefined("implements", Type.LABEL_LIST)) { + builder.addTargets(context.getPrerequisites("implements", Mode.TARGET), + RunfilesProvider.DEFAULT_RUNFILES); + builder.addTargets(context.getPrerequisites("implements", Mode.TARGET), + CppRunfilesProvider.runfilesFunction(linkingStatically)); + } + if (context.getRule().isAttrDefined("implementation", Type.LABEL_LIST)) { + builder.addTargets(context.getPrerequisites("implementation", Mode.TARGET), + RunfilesProvider.DEFAULT_RUNFILES); + builder.addTargets(context.getPrerequisites("implementation", Mode.TARGET), + CppRunfilesProvider.runfilesFunction(linkingStatically)); + } + + builder.addDataDeps(context); + + if (addDynamicRuntimeInputArtifactsToRunfiles) { + builder.addTransitiveArtifacts(CppHelper.getToolchain(context).getDynamicRuntimeLinkInputs()); + } + return builder.build(); + } + + @Override + public ConfiguredTarget create(RuleContext context) { + RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(context); + LinkTargetType linkType = getStaticLinkType(context); + boolean linkStatic = context.attributes().get("linkstatic", Type.BOOLEAN); + init(semantics, context, builder, linkType, + /*neverLink =*/ false, + linkStatic, + /*collectLinkstamp =*/ true, + /*addDynamicRuntimeInputArtifactsToRunfiles =*/ false); + return builder.build(); + } + + public static void init(CppSemantics semantics, RuleContext ruleContext, + RuleConfiguredTargetBuilder targetBuilder, LinkTargetType linkType, + boolean neverLink, + boolean linkStatic, + boolean collectLinkstamp, + boolean addDynamicRuntimeInputArtifactsToRunfiles) { + final CcCommon common = new CcCommon(ruleContext); + + CcLibraryHelper helper = new CcLibraryHelper(ruleContext, semantics) + .setLinkType(linkType) + .enableCcNativeLibrariesProvider() + .enableInterfaceSharedObjects() + .enableCompileProviders() + .setNeverLink(neverLink) + .setHeadersCheckingMode(common.determineHeadersCheckingMode()) + .addCopts(common.getCopts()) + .setNoCopts(common.getNoCopts()) + .addLinkopts(common.getLinkopts()) + .addDefines(common.getDefines()) + .addCompilationPrerequisites(common.getSharedLibrariesFromSrcs()) + .addCompilationPrerequisites(common.getStaticLibrariesFromSrcs()) + .addSources(common.getCAndCppSources()) + .addPublicHeaders(common.getHeaders()) + .addObjectFiles(common.getObjectFilesFromSrcs(false)) + .addPicObjectFiles(common.getObjectFilesFromSrcs(true)) + .addPicIndependentObjectFiles(common.getLinkerScripts()) + .addDeps(ruleContext.getPrerequisites("deps", Mode.TARGET)) + .setEnableLayeringCheck(ruleContext.getFeatures().contains(CppRuleClasses.LAYERING_CHECK)) + .setCompileHeaderModules(ruleContext.getFeatures().contains(CppRuleClasses.HEADER_MODULES)) + .addSystemIncludeDirs(common.getSystemIncludeDirs()) + .addIncludeDirs(common.getIncludeDirs()) + .addLooseIncludeDirs(common.getLooseIncludeDirs()) + .setEmitHeaderTargetModuleMaps( + ruleContext.getRule().getRuleClass().equals("cc_public_library")); + + if (collectLinkstamp) { + helper.addLinkstamps(ruleContext.getPrerequisites("linkstamp", Mode.TARGET)); + } + + if (ruleContext.getRule().isAttrDefined("implements", Type.LABEL_LIST)) { + helper.addDeps(ruleContext.getPrerequisites("implements", Mode.TARGET)); + } + + if (ruleContext.getRule().isAttrDefined("implementation", Type.LABEL_LIST)) { + helper.addDeps(ruleContext.getPrerequisites("implementation", Mode.TARGET)); + } + + PathFragment soImplFilename = null; + if (ruleContext.getRule().isAttrDefined("outs", Type.STRING_LIST)) { + List<String> outs = ruleContext.attributes().get("outs", Type.STRING_LIST); + if (outs.size() > 1) { + ruleContext.attributeError("outs", "must be a singleton list"); + } else if (outs.size() == 1) { + soImplFilename = CppHelper.getLinkedFilename(ruleContext, LinkTargetType.DYNAMIC_LIBRARY); + soImplFilename = soImplFilename.replaceName(outs.get(0)); + if (!soImplFilename.getPathString().endsWith(".so")) { // Sanity check. + ruleContext.attributeError("outs", "file name must end in '.so'"); + } + } + } + + if (ruleContext.getRule().isAttrDefined("srcs", Type.LABEL_LIST)) { + helper.addPrivateHeaders(FileType.filter( + ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list(), + CppFileTypes.CPP_HEADER)); + ruleContext.checkSrcsSamePackage(true); + } + + if (common.getLinkopts().contains("-static")) { + ruleContext.attributeWarning("linkopts", "Using '-static' here won't work. " + + "Did you mean to use 'linkstatic=1' instead?"); + } + + boolean createDynamicLibrary = + !linkStatic && !appearsToHaveNoObjectFiles(ruleContext.attributes()); + helper.setCreateDynamicLibrary(createDynamicLibrary); + helper.setDynamicLibraryPath(soImplFilename); + + /* + * Add the libraries from srcs, if any. For static/mostly static + * linking we setup the dynamic libraries if there are no static libraries + * to choose from. Path to the libraries will be mangled to avoid using + * absolute path names on the -rpath, but library filenames will be + * preserved (since some libraries might have SONAME tag) - symlink will + * be created to the parent directory instead. + * + * For compatibility with existing BUILD files, any ".a" or ".lo" files listed in + * srcs are assumed to be position-independent code, or at least suitable for + * inclusion in shared libraries, unless they end with ".nopic.a" or ".nopic.lo". + * + * Note that some target platforms do not require shared library code to be PIC. + */ + Iterable<LibraryToLink> staticLibrariesFromSrcs = + LinkerInputs.opaqueLibrariesToLink(common.getStaticLibrariesFromSrcs()); + helper.addStaticLibraries(staticLibrariesFromSrcs); + helper.addPicStaticLibraries(Iterables.filter(staticLibrariesFromSrcs, PIC_STATIC_FILTER)); + helper.addPicStaticLibraries(common.getPicStaticLibrariesFromSrcs()); + helper.addDynamicLibraries(Iterables.transform(common.getSharedLibrariesFromSrcs(), + new Function<Artifact, LibraryToLink>() { + @Override + public LibraryToLink apply(Artifact library) { + return common.getDynamicLibrarySymlink(library, true); + } + })); + CcLibraryHelper.Info info = helper.build(); + + /* + * We always generate a static library, even if there aren't any source files. + * This keeps things simpler by avoiding special cases when making use of the library. + * For example, this is needed to ensure that building a library with "bazel build" + * will also build all of the library's "deps". + * However, we only generate a dynamic library if there are source files. + */ + // For now, we don't add the precompiled libraries to the files to build. + CcLinkingOutputs linkedLibraries = info.getCcLinkingOutputsExcludingPrecompiledLibraries(); + + NestedSet<Artifact> artifactsToForce = + collectArtifactsToForce(ruleContext, common, info.getCcCompilationOutputs()); + + NestedSetBuilder<Artifact> filesBuilder = NestedSetBuilder.stableOrder(); + filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkedLibraries.getStaticLibraries())); + filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkedLibraries.getPicStaticLibraries())); + filesBuilder.addAll(LinkerInputs.toNonSolibArtifacts(linkedLibraries.getDynamicLibraries())); + filesBuilder.addAll( + LinkerInputs.toNonSolibArtifacts(linkedLibraries.getExecutionDynamicLibraries())); + + CcLinkingOutputs linkingOutputs = info.getCcLinkingOutputs(); + warnAboutEmptyLibraries( + ruleContext, info.getCcCompilationOutputs(), linkType, linkStatic); + NestedSet<Artifact> filesToBuild = filesBuilder.build(); + + Runfiles staticRunfiles = collectRunfiles(ruleContext, + linkingOutputs, neverLink, addDynamicRuntimeInputArtifactsToRunfiles, true); + Runfiles sharedRunfiles = collectRunfiles(ruleContext, + linkingOutputs, neverLink, addDynamicRuntimeInputArtifactsToRunfiles, false); + + List<Artifact> instrumentedObjectFiles = new ArrayList<>(); + instrumentedObjectFiles.addAll(info.getCcCompilationOutputs().getObjectFiles(false)); + instrumentedObjectFiles.addAll(info.getCcCompilationOutputs().getObjectFiles(true)); + InstrumentedFilesProvider instrumentedFilesProvider = + common.getInstrumentedFilesProvider(instrumentedObjectFiles); + targetBuilder + .setFilesToBuild(filesToBuild) + .addProviders(info.getProviders()) + .add(InstrumentedFilesProvider.class, instrumentedFilesProvider) + .add(RunfilesProvider.class, RunfilesProvider.withData(staticRunfiles, sharedRunfiles)) + // Remove this? + .add(CppRunfilesProvider.class, new CppRunfilesProvider(staticRunfiles, sharedRunfiles)) + .setBaselineCoverageArtifacts(BaselineCoverageAction.getBaselineCoverageArtifacts( + ruleContext, instrumentedFilesProvider.getInstrumentedFiles())) + .add(ImplementedCcPublicLibrariesProvider.class, + new ImplementedCcPublicLibrariesProvider(getImplementedCcPublicLibraries(ruleContext))) + .add(AlwaysBuiltArtifactsProvider.class, + new AlwaysBuiltArtifactsProvider(artifactsToForce)); + } + + private static NestedSet<Artifact> collectArtifactsToForce(RuleContext ruleContext, + CcCommon common, CcCompilationOutputs ccCompilationOutputs) { + // Ensure that we build all the dependencies, otherwise users may get confused. + NestedSetBuilder<Artifact> artifactsToForceBuilder = NestedSetBuilder.stableOrder(); + artifactsToForceBuilder.addTransitive( + NestedSetBuilder.wrap(Order.STABLE_ORDER, common.getFilesToCompile(ccCompilationOutputs))); + for (AlwaysBuiltArtifactsProvider dep : + ruleContext.getPrerequisites("deps", Mode.TARGET, AlwaysBuiltArtifactsProvider.class)) { + artifactsToForceBuilder.addTransitive(dep.getArtifactsToAlwaysBuild()); + } + return artifactsToForceBuilder.build(); + } + + /** + * Returns the type of the generated static library. + */ + private static LinkTargetType getStaticLinkType(RuleContext context) { + return context.attributes().get("alwayslink", Type.BOOLEAN) + ? LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY + : LinkTargetType.STATIC_LIBRARY; + } + + private static void warnAboutEmptyLibraries(RuleContext ruleContext, + CcCompilationOutputs ccCompilationOutputs, LinkTargetType linkType, + boolean linkstaticAttribute) { + if (ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector()) { + // Do not signal warnings in the lipo context collector configuration. These will be duly + // signaled in the target configuration, and there can be spurious warnings since targets in + // the LIPO context collector configuration do not compile anything. + return; + } + if (ccCompilationOutputs.getObjectFiles(false).isEmpty() + && ccCompilationOutputs.getObjectFiles(true).isEmpty()) { + if (linkType == LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY + || linkType == LinkTargetType.ALWAYS_LINK_PIC_STATIC_LIBRARY) { + ruleContext.attributeWarning("alwayslink", + "'alwayslink' has no effect if there are no 'srcs'"); + } + if (!linkstaticAttribute && !appearsToHaveNoObjectFiles(ruleContext.attributes())) { + ruleContext.attributeWarning("linkstatic", + "setting 'linkstatic=1' is recommended if there are no object files"); + } + } else { + if (!linkstaticAttribute && appearsToHaveNoObjectFiles(ruleContext.attributes())) { + Artifact element = ccCompilationOutputs.getObjectFiles(false).isEmpty() + ? ccCompilationOutputs.getObjectFiles(true).get(0) + : ccCompilationOutputs.getObjectFiles(false).get(0); + ruleContext.attributeWarning("srcs", + "this library appears at first glance to have no object files, " + + "but on closer inspection it does have something to link, e.g. " + + element.prettyPrint() + ". " + + "(You may have used some very confusing rule names in srcs? " + + "Or the library consists entirely of a linker script?) " + + "Bazel assumed linkstatic=1, but this may be inappropriate. " + + "You may need to add an explicit '.cc' file to 'srcs'. " + + "Alternatively, add 'linkstatic=1' to suppress this warning"); + } + } + } + + private static ImmutableList<Label> getImplementedCcPublicLibraries(RuleContext context) { + if (context.getRule().getRuleClassObject().hasAttr("implements", Type.LABEL_LIST)) { + return ImmutableList.copyOf(context.attributes().get("implements", Type.LABEL_LIST)); + } else { + return ImmutableList.of(); + } + } + + /** + * Returns true if the rule (which must be a cc_library rule) + * appears to have no object files. This only looks at the rule + * itself, not at any other rules (from this package or other + * packages) that it might reference. + * + * <p> + * In some cases, this may return "false" even + * though the rule actually has no object files. + * For example, it will return false for a rule such as + * <code>cc_library(name = 'foo', srcs = [':bar'])</code> + * because we can't tell what ':bar' is; it might + * be a genrule that generates a source file, or it might + * be a genrule that generates a header file. + * + * <p> + * In other cases, this may return "true" even + * though the rule actually does have object files. + * For example, it will return true for a rule such as + * <code>cc_library(name = 'foo', srcs = ['bar.h'])</code> + * but as in the other example above, we can't tell whether + * 'bar.h' is a file name or a rule name, and 'bar.h' could + * in fact be the name of a genrule that generates a source file. + */ + public static boolean appearsToHaveNoObjectFiles(AttributeMap rule) { + // Temporary hack while configurable attributes is under development. This has no effect + // for any rule that doesn't use configurable attributes. + // TODO(bazel-team): remove this hack for a more principled solution. + try { + rule.get("srcs", Type.LABEL_LIST); + } catch (ClassCastException e) { + // "srcs" is actually a configurable selector. Assume object files are possible somewhere. + return false; + } + + List<Label> srcs = rule.get("srcs", Type.LABEL_LIST); + if (srcs != null) { + for (Label srcfile : srcs) { + /* + * We cheat a little bit here by looking at the file extension + * of the Label treated as file name. In general that might + * not necessarily work, because of the possibility that the + * user might give a rule a funky name ending in one of these + * extensions, e.g. + * genrule(name = 'foo.h', outs = ['foo.cc'], ...) // Funky rule name! + * cc_library(name = 'bar', srcs = ['foo.h']) // This DOES have object files. + */ + if (!NO_OBJECT_GENERATING_FILETYPES.matches(srcfile.getName())) { + return false; + } + } + } + return true; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibraryHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibraryHelper.java new file mode 100644 index 0000000000..3812bf5001 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibraryHelper.java @@ -0,0 +1,905 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.CompilationPrerequisitesProvider; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToCompileProvider; +import com.google.devtools.build.lib.analysis.LanguageDependentFragment; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TempsProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.HeadersCheckingMode; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +/** + * A class to create C/C++ compile and link actions in a way that is consistent with cc_library. + * Rules that generate source files and emulate cc_library on top of that should use this class + * instead of the lower-level APIs in CppHelper and CppModel. + * + * <p>Rules that want to use this class are required to have implicit dependencies on the + * toolchain, the STL, the lipo context, and so on. Optionally, they can also have copts, + * and malloc attributes, but note that these require explicit calls to the corresponding setter + * methods. + */ +public final class CcLibraryHelper { + /** Function for extracting module maps from CppCompilationDependencies. */ + public static final Function<TransitiveInfoCollection, CppModuleMap> CPP_DEPS_TO_MODULES = + new Function<TransitiveInfoCollection, CppModuleMap>() { + @Override + @Nullable + public CppModuleMap apply(TransitiveInfoCollection dep) { + CppCompilationContext context = dep.getProvider(CppCompilationContext.class); + return context == null ? null : context.getCppModuleMap(); + } + }; + + /** + * Contains the providers as well as the compilation and linking outputs, and the compilation + * context. + */ + public static final class Info { + private final Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers; + private final CcCompilationOutputs compilationOutputs; + private final CcLinkingOutputs linkingOutputs; + private final CcLinkingOutputs linkingOutputsExcludingPrecompiledLibraries; + private final CppCompilationContext context; + + private Info(Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers, + CcCompilationOutputs compilationOutputs, CcLinkingOutputs linkingOutputs, + CcLinkingOutputs linkingOutputsExcludingPrecompiledLibraries, + CppCompilationContext context) { + this.providers = Collections.unmodifiableMap(providers); + this.compilationOutputs = compilationOutputs; + this.linkingOutputs = linkingOutputs; + this.linkingOutputsExcludingPrecompiledLibraries = + linkingOutputsExcludingPrecompiledLibraries; + this.context = context; + } + + public Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> getProviders() { + return providers; + } + + public CcCompilationOutputs getCcCompilationOutputs() { + return compilationOutputs; + } + + public CcLinkingOutputs getCcLinkingOutputs() { + return linkingOutputs; + } + + /** + * Returns the linking outputs before adding the pre-compiled libraries. Avoid using this - + * pre-compiled and locally compiled libraries should be treated identically. This method only + * exists for backwards compatibility. + */ + public CcLinkingOutputs getCcLinkingOutputsExcludingPrecompiledLibraries() { + return linkingOutputsExcludingPrecompiledLibraries; + } + + public CppCompilationContext getCppCompilationContext() { + return context; + } + + /** + * Adds the static, pic-static, and dynamic (both compile-time and execution-time) libraries to + * the given builder. + */ + public void addLinkingOutputsTo(NestedSetBuilder<Artifact> filesBuilder) { + filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkingOutputs.getStaticLibraries())); + filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkingOutputs.getPicStaticLibraries())); + filesBuilder.addAll(LinkerInputs.toNonSolibArtifacts(linkingOutputs.getDynamicLibraries())); + filesBuilder.addAll( + LinkerInputs.toNonSolibArtifacts(linkingOutputs.getExecutionDynamicLibraries())); + } + } + + private final RuleContext ruleContext; + private final BuildConfiguration configuration; + private final CppSemantics semantics; + + private final List<Artifact> publicHeaders = new ArrayList<>(); + private final List<Artifact> privateHeaders = new ArrayList<>(); + private final List<PathFragment> additionalExportedHeaders = new ArrayList<>(); + private final List<Pair<Artifact, Label>> sources = new ArrayList<>(); + private final List<Artifact> objectFiles = new ArrayList<>(); + private final List<Artifact> picObjectFiles = new ArrayList<>(); + private final List<String> copts = new ArrayList<>(); + @Nullable private Pattern nocopts; + private final List<String> linkopts = new ArrayList<>(); + private final Set<String> defines = new LinkedHashSet<>(); + private final List<TransitiveInfoCollection> deps = new ArrayList<>(); + private final List<Artifact> linkstamps = new ArrayList<>(); + private final List<Artifact> prerequisites = new ArrayList<>(); + private final List<PathFragment> looseIncludeDirs = new ArrayList<>(); + private final List<PathFragment> systemIncludeDirs = new ArrayList<>(); + private final List<PathFragment> includeDirs = new ArrayList<>(); + @Nullable private PathFragment dynamicLibraryPath; + private LinkTargetType linkType = LinkTargetType.STATIC_LIBRARY; + private HeadersCheckingMode headersCheckingMode = HeadersCheckingMode.LOOSE; + private boolean neverlink; + private boolean fake; + + private final List<LibraryToLink> staticLibraries = new ArrayList<>(); + private final List<LibraryToLink> picStaticLibraries = new ArrayList<>(); + private final List<LibraryToLink> dynamicLibraries = new ArrayList<>(); + + private boolean emitCppModuleMaps = true; + private boolean enableLayeringCheck; + private boolean compileHeaderModules; + private boolean emitCompileActionsIfEmpty = true; + private boolean emitCcNativeLibrariesProvider; + private boolean emitCcSpecificLinkParamsProvider; + private boolean emitInterfaceSharedObjects; + private boolean emitDynamicLibrary = true; + private boolean checkDepsGenerateCpp = true; + private boolean emitCompileProviders; + private boolean emitHeaderTargetModuleMaps = false; + + public CcLibraryHelper(RuleContext ruleContext, CppSemantics semantics) { + this.ruleContext = Preconditions.checkNotNull(ruleContext); + this.configuration = ruleContext.getConfiguration(); + this.semantics = Preconditions.checkNotNull(semantics); + } + + /** + * Add the corresponding files as header files, i.e., these files will not be compiled, but are + * made visible as includes to dependent rules. + */ + public CcLibraryHelper addPublicHeaders(Collection<Artifact> headers) { + this.publicHeaders.addAll(headers); + return this; + } + + /** + * Add the corresponding files as public header files, i.e., these files will not be compiled, but + * are made visible as includes to dependent rules in module maps. + */ + public CcLibraryHelper addPublicHeaders(Artifact... headers) { + return addPublicHeaders(Arrays.asList(headers)); + } + + /** + * Add the corresponding files as private header files, i.e., these files will not be compiled, + * but are not made visible as includes to dependent rules in module maps. + */ + public CcLibraryHelper addPrivateHeaders(Iterable<Artifact> privateHeaders) { + Iterables.addAll(this.privateHeaders, privateHeaders); + return this; + } + + /** + * Add the corresponding files as public header files, i.e., these files will not be compiled, but + * are made visible as includes to dependent rules in module maps. + */ + public CcLibraryHelper addAdditionalExportedHeaders( + Iterable<PathFragment> additionalExportedHeaders) { + Iterables.addAll(this.additionalExportedHeaders, additionalExportedHeaders); + return this; + } + + /** + * Add the corresponding files as source files. These may also be header files, in which case + * they will not be compiled, but also not made visible as includes to dependent rules. + */ + // TODO(bazel-team): This is inconsistent with the documentation on CppModel. + public CcLibraryHelper addSources(Collection<Artifact> sources) { + for (Artifact source : sources) { + this.sources.add(Pair.of(source, ruleContext.getLabel())); + } + return this; + } + + /** + * Add the corresponding files as source files. These may also be header files, in which case + * they will not be compiled, but also not made visible as includes to dependent rules. + */ + // TODO(bazel-team): This is inconsistent with the documentation on CppModel. + public CcLibraryHelper addSources(Iterable<Pair<Artifact, Label>> sources) { + Iterables.addAll(this.sources, sources); + return this; + } + + /** + * Add the corresponding files as source files. These may also be header files, in which case + * they will not be compiled, but also not made visible as includes to dependent rules. + */ + public CcLibraryHelper addSources(Artifact... sources) { + return addSources(Arrays.asList(sources)); + } + + /** + * Add the corresponding files as linker inputs for non-PIC links. If the corresponding files are + * compiled with PIC, the final link may or may not fail. Note that the final link may not happen + * here, if {@code --start_end_lib} is enabled, but instead at any binary that transitively + * depends on the current rule. + */ + public CcLibraryHelper addObjectFiles(Iterable<Artifact> objectFiles) { + Iterables.addAll(this.objectFiles, objectFiles); + return this; + } + + /** + * Add the corresponding files as linker inputs for PIC links. If the corresponding files are not + * compiled with PIC, the final link may or may not fail. Note that the final link may not happen + * here, if {@code --start_end_lib} is enabled, but instead at any binary that transitively + * depends on the current rule. + */ + public CcLibraryHelper addPicObjectFiles(Iterable<Artifact> picObjectFiles) { + Iterables.addAll(this.picObjectFiles, picObjectFiles); + return this; + } + + /** + * Add the corresponding files as linker inputs for both PIC and non-PIC links. + */ + public CcLibraryHelper addPicIndependentObjectFiles(Iterable<Artifact> objectFiles) { + addPicObjectFiles(objectFiles); + return addObjectFiles(objectFiles); + } + + /** + * Add the corresponding files as linker inputs for both PIC and non-PIC links. + */ + public CcLibraryHelper addPicIndependentObjectFiles(Artifact... objectFiles) { + return addPicIndependentObjectFiles(Arrays.asList(objectFiles)); + } + + /** + * Add the corresponding files as static libraries into the linker outputs (i.e., after the linker + * action) - this makes them available for linking to binary rules that depend on this rule. + */ + public CcLibraryHelper addStaticLibraries(Iterable<LibraryToLink> libraries) { + Iterables.addAll(staticLibraries, libraries); + return this; + } + + /** + * Add the corresponding files as static libraries into the linker outputs (i.e., after the linker + * action) - this makes them available for linking to binary rules that depend on this rule. + */ + public CcLibraryHelper addPicStaticLibraries(Iterable<LibraryToLink> libraries) { + Iterables.addAll(picStaticLibraries, libraries); + return this; + } + + /** + * Add the corresponding files as dynamic libraries into the linker outputs (i.e., after the + * linker action) - this makes them available for linking to binary rules that depend on this + * rule. + */ + public CcLibraryHelper addDynamicLibraries(Iterable<LibraryToLink> libraries) { + Iterables.addAll(dynamicLibraries, libraries); + return this; + } + + /** + * Adds the copts to the compile command line. + */ + public CcLibraryHelper addCopts(Iterable<String> copts) { + Iterables.addAll(this.copts, copts); + return this; + } + + /** + * Sets a pattern that is used to filter copts; set to {@code null} for no filtering. + */ + public CcLibraryHelper setNoCopts(@Nullable Pattern nocopts) { + this.nocopts = nocopts; + return this; + } + + /** + * Adds the given options as linker options to the link command. + */ + public CcLibraryHelper addLinkopts(Iterable<String> linkopts) { + Iterables.addAll(this.linkopts, linkopts); + return this; + } + + /** + * Adds the given defines to the compiler command line. + */ + public CcLibraryHelper addDefines(Iterable<String> defines) { + Iterables.addAll(this.defines, defines); + return this; + } + + /** + * Adds the given targets as dependencies - this can include explicit dependencies on other + * rules (like from a "deps" attribute) and also implicit dependencies on runtime libraries. + */ + public CcLibraryHelper addDeps(Iterable<? extends TransitiveInfoCollection> deps) { + for (TransitiveInfoCollection dep : deps) { + Preconditions.checkArgument(dep.getConfiguration() == null + || dep.getConfiguration().equals(configuration)); + this.deps.add(dep); + } + return this; + } + + /** + * Adds the given linkstamps. Note that linkstamps are usually not compiled at the library level, + * but only in the dependent binary rules. + */ + public CcLibraryHelper addLinkstamps(Iterable<? extends TransitiveInfoCollection> linkstamps) { + for (TransitiveInfoCollection linkstamp : linkstamps) { + Iterables.addAll(this.linkstamps, + linkstamp.getProvider(FileProvider.class).getFilesToBuild()); + } + return this; + } + + /** + * Adds the given prerequisites as prerequisites for the generated compile actions. This ensures + * that the corresponding files exist - otherwise the action fails. Note that these dependencies + * add edges to the action graph, and can therefore increase the length of the critical path, + * i.e., make the build slower. + */ + public CcLibraryHelper addCompilationPrerequisites(Iterable<Artifact> prerequisites) { + Iterables.addAll(this.prerequisites, prerequisites); + return this; + } + + /** + * Adds the given directories to the loose include directories that are only allowed to be + * referenced when headers checking is {@link HeadersCheckingMode#LOOSE} or {@link + * HeadersCheckingMode#WARN}. + */ + public CcLibraryHelper addLooseIncludeDirs(Iterable<PathFragment> looseIncludeDirs) { + Iterables.addAll(this.looseIncludeDirs, looseIncludeDirs); + return this; + } + + /** + * Adds the given directories to the system include directories (they are passed with {@code + * "-isystem"} to the compiler); these are also passed to dependent rules. + */ + public CcLibraryHelper addSystemIncludeDirs(Iterable<PathFragment> systemIncludeDirs) { + Iterables.addAll(this.systemIncludeDirs, systemIncludeDirs); + return this; + } + + /** + * Adds the given directories to the quote include directories (they are passed with {@code + * "-iquote"} to the compiler); these are also passed to dependent rules. + */ + public CcLibraryHelper addIncludeDirs(Iterable<PathFragment> includeDirs) { + Iterables.addAll(this.includeDirs, includeDirs); + return this; + } + + /** + * Overrides the path for the generated dynamic library - this should only be called if the + * dynamic library is an implicit or explicit output of the rule, i.e., if it is accessible by + * name from other rules in the same package. Set to {@code null} to use the default computation. + */ + public CcLibraryHelper setDynamicLibraryPath(@Nullable PathFragment dynamicLibraryPath) { + this.dynamicLibraryPath = dynamicLibraryPath; + return this; + } + + /** + * Marks the output of this rule as alwayslink, i.e., the corresponding symbols will be retained + * by the linker even if they are not otherwise used. This is useful for libraries that register + * themselves somewhere during initialization. + * + * <p>This only sets the link type (see {@link #setLinkType}), either to a static library or to + * an alwayslink static library (blaze uses a different file extension to signal alwayslink to + * downstream code). + */ + public CcLibraryHelper setAlwayslink(boolean alwayslink) { + linkType = alwayslink + ? LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY + : LinkTargetType.STATIC_LIBRARY; + return this; + } + + /** + * Directly set the link type. This can be used instead of {@link #setAlwayslink}. Setting + * anything other than a static link causes this class to skip the link action creation. + */ + public CcLibraryHelper setLinkType(LinkTargetType linkType) { + this.linkType = Preconditions.checkNotNull(linkType); + return this; + } + + /** + * Marks the resulting code as neverlink, i.e., the code will not be linked into dependent + * libraries or binaries - the header files are still available. + */ + public CcLibraryHelper setNeverLink(boolean neverlink) { + this.neverlink = neverlink; + return this; + } + + /** + * Sets the given headers checking mode. The default is {@link HeadersCheckingMode#LOOSE}. + */ + public CcLibraryHelper setHeadersCheckingMode(HeadersCheckingMode headersCheckingMode) { + this.headersCheckingMode = Preconditions.checkNotNull(headersCheckingMode); + return this; + } + + /** + * Marks the resulting code as fake, i.e., the code will not actually be compiled or linked, but + * instead, the compile command is written to a file and added to the runfiles. This is currently + * used for non-compilation tests. Unfortunately, the design is problematic, so please don't add + * any further uses. + */ + public CcLibraryHelper setFake(boolean fake) { + this.fake = fake; + return this; + } + + /** + * This adds the {@link CcNativeLibraryProvider} to the providers created by this class. + */ + public CcLibraryHelper enableCcNativeLibrariesProvider() { + this.emitCcNativeLibrariesProvider = true; + return this; + } + + /** + * This adds the {@link CcSpecificLinkParamsProvider} to the providers created by this class. + * Otherwise the result will contain an instance of {@link CcLinkParamsProvider}. + */ + public CcLibraryHelper enableCcSpecificLinkParamsProvider() { + this.emitCcSpecificLinkParamsProvider = true; + return this; + } + + /** + * This disables C++ module map generation for the current rule. Don't call this unless you know + * what you are doing. + */ + public CcLibraryHelper disableCppModuleMapGeneration() { + this.emitCppModuleMaps = false; + return this; + } + + /** + * This enables or disables use of module maps during compilation, i.e., layering checks. + */ + public CcLibraryHelper setEnableLayeringCheck(boolean enableLayeringCheck) { + this.enableLayeringCheck = enableLayeringCheck; + return this; + } + + /** + * This enabled or disables compilation of C++ header modules. + * TODO(bazel-team): Add a cc_toolchain flag that allows fully disabling this feature and document + * this feature. + * See http://clang.llvm.org/docs/Modules.html. + */ + public CcLibraryHelper setCompileHeaderModules(boolean compileHeaderModules) { + this.compileHeaderModules = compileHeaderModules; + return this; + } + + /** + * Enables or disables generation of compile actions if there are no sources. Some rules declare a + * .a or .so implicit output, which requires that these files are created even if there are no + * source files, so be careful when calling this. + */ + public CcLibraryHelper setGenerateCompileActionsIfEmpty(boolean emitCompileActionsIfEmpty) { + this.emitCompileActionsIfEmpty = emitCompileActionsIfEmpty; + return this; + } + + /** + * Enables the optional generation of interface dynamic libraries - this is only used when the + * linker generates a dynamic library, and only if the crosstool supports it. The default is not + * to generate interface dynamic libraries. + */ + public CcLibraryHelper enableInterfaceSharedObjects() { + this.emitInterfaceSharedObjects = true; + return this; + } + + /** + * This enables or disables the generation of a dynamic library link action. The default is to + * generate a dynamic library. Note that the selection between dynamic or static linking is + * performed at the binary rule level. + */ + public CcLibraryHelper setCreateDynamicLibrary(boolean emitDynamicLibrary) { + this.emitDynamicLibrary = emitDynamicLibrary; + return this; + } + + /** + * Disables checking that the deps actually are C++ rules. By default, the {@link #build} method + * uses {@link LanguageDependentFragment.Checker#depSupportsLanguage} to check that all deps + * provide C++ providers. + */ + public CcLibraryHelper setCheckDepsGenerateCpp(boolean checkDepsGenerateCpp) { + this.checkDepsGenerateCpp = checkDepsGenerateCpp; + return this; + } + + /** + * Enables the output of {@link FilesToCompileProvider} and {@link + * CompilationPrerequisitesProvider}. + */ + // TODO(bazel-team): We probably need to adjust this for the multi-language rules. + public CcLibraryHelper enableCompileProviders() { + this.emitCompileProviders = true; + return this; + } + + /** + * Sets whether to emit the transitive module map references of a public library headers target. + */ + public CcLibraryHelper setEmitHeaderTargetModuleMaps(boolean emitHeaderTargetModuleMaps) { + this.emitHeaderTargetModuleMaps = emitHeaderTargetModuleMaps; + return this; + } + + /** + * Create the C++ compile and link actions, and the corresponding C++-related providers. + */ + public Info build() { + // Fail early if there is no lipo context collector on the rule - otherwise we end up failing + // in lipo optimization. + Preconditions.checkState( + // 'cc_inc_library' rules do not compile, and thus are not affected by LIPO. + ruleContext.getRule().getRuleClass().equals("cc_inc_library") + || ruleContext.getRule().isAttrDefined(":lipo_context_collector", Type.LABEL)); + + if (checkDepsGenerateCpp) { + for (LanguageDependentFragment dep : + AnalysisUtils.getProviders(deps, LanguageDependentFragment.class)) { + LanguageDependentFragment.Checker.depSupportsLanguage( + ruleContext, dep, CppRuleClasses.LANGUAGE); + } + } + + CcLinkingOutputs ccLinkingOutputs = CcLinkingOutputs.EMPTY; + CcCompilationOutputs ccOutputs = new CcCompilationOutputs.Builder().build(); + FeatureConfiguration featureConfiguration = CcCommon.configureFeatures(ruleContext); + + CppModel model = new CppModel(ruleContext, semantics) + .addSources(sources) + .addCopts(copts) + .setLinkTargetType(linkType) + .setNeverLink(neverlink) + .setFake(fake) + .setAllowInterfaceSharedObjects(emitInterfaceSharedObjects) + .setCreateDynamicLibrary(emitDynamicLibrary) + // Note: this doesn't actually save the temps, it just makes the CppModel use the + // configurations --save_temps setting to decide whether to actually save the temps. + .setSaveTemps(true) + .setEnableLayeringCheck(enableLayeringCheck) + .setCompileHeaderModules(compileHeaderModules) + .setNoCopts(nocopts) + .setDynamicLibraryPath(dynamicLibraryPath) + .addLinkopts(linkopts) + .setFeatureConfiguration(featureConfiguration); + CppCompilationContext cppCompilationContext = + initializeCppCompilationContext(model, featureConfiguration); + model.setContext(cppCompilationContext); + if (emitCompileActionsIfEmpty || !sources.isEmpty() || compileHeaderModules) { + Preconditions.checkState( + !compileHeaderModules || cppCompilationContext.getCppModuleMap() != null, + "All cc rules must support module maps."); + ccOutputs = model.createCcCompileActions(); + if (!objectFiles.isEmpty() || !picObjectFiles.isEmpty()) { + // Merge the pre-compiled object files into the compiler outputs. + ccOutputs = new CcCompilationOutputs.Builder() + .merge(ccOutputs) + .addObjectFiles(objectFiles) + .addPicObjectFiles(picObjectFiles) + .build(); + } + if (linkType.isStaticLibraryLink()) { + // TODO(bazel-team): This can't create the link action for a cc_binary yet. + ccLinkingOutputs = model.createCcLinkActions(ccOutputs); + } + } + CcLinkingOutputs originalLinkingOutputs = ccLinkingOutputs; + if (!( + staticLibraries.isEmpty() && picStaticLibraries.isEmpty() && dynamicLibraries.isEmpty())) { + // Merge the pre-compiled libraries (static & dynamic) into the linker outputs. + ccLinkingOutputs = new CcLinkingOutputs.Builder() + .merge(ccLinkingOutputs) + .addStaticLibraries(staticLibraries) + .addPicStaticLibraries(picStaticLibraries) + .addDynamicLibraries(dynamicLibraries) + .addExecutionDynamicLibraries(dynamicLibraries) + .build(); + } + + DwoArtifactsCollector dwoArtifacts = DwoArtifactsCollector.transitiveCollector(ccOutputs, deps); + Runfiles cppStaticRunfiles = collectCppRunfiles(ccLinkingOutputs, true); + Runfiles cppSharedRunfiles = collectCppRunfiles(ccLinkingOutputs, false); + + // By very careful when adding new providers here - it can potentially affect a lot of rules. + // We should consider merging most of these providers into a single provider. + Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers = + new LinkedHashMap<>(); + providers.put(CppRunfilesProvider.class, + new CppRunfilesProvider(cppStaticRunfiles, cppSharedRunfiles)); + providers.put(CppCompilationContext.class, cppCompilationContext); + providers.put(CppDebugFileProvider.class, new CppDebugFileProvider( + dwoArtifacts.getDwoArtifacts(), dwoArtifacts.getPicDwoArtifacts())); + providers.put(TransitiveLipoInfoProvider.class, collectTransitiveLipoInfo(ccOutputs)); + providers.put(TempsProvider.class, getTemps(ccOutputs)); + if (emitCompileProviders) { + providers.put(FilesToCompileProvider.class, new FilesToCompileProvider( + getFilesToCompile(ccOutputs))); + providers.put(CompilationPrerequisitesProvider.class, + CcCommon.collectCompilationPrerequisites(ruleContext, cppCompilationContext)); + } + + // TODO(bazel-team): Maybe we can infer these from other data at the places where they are + // used. + if (emitCcNativeLibrariesProvider) { + providers.put(CcNativeLibraryProvider.class, + new CcNativeLibraryProvider(collectNativeCcLibraries(ccLinkingOutputs))); + } + providers.put(CcExecutionDynamicLibrariesProvider.class, + collectExecutionDynamicLibraryArtifacts(ccLinkingOutputs.getExecutionDynamicLibraries())); + + boolean forcePic = ruleContext.getFragment(CppConfiguration.class).forcePic(); + if (emitCcSpecificLinkParamsProvider) { + providers.put(CcSpecificLinkParamsProvider.class, new CcSpecificLinkParamsProvider( + createCcLinkParamsStore(ccLinkingOutputs, cppCompilationContext, forcePic))); + } else { + providers.put(CcLinkParamsProvider.class, new CcLinkParamsProvider( + createCcLinkParamsStore(ccLinkingOutputs, cppCompilationContext, forcePic))); + } + return new Info(providers, ccOutputs, ccLinkingOutputs, originalLinkingOutputs, + cppCompilationContext); + } + + /** + * Create context for cc compile action from generated inputs. + */ + private CppCompilationContext initializeCppCompilationContext(CppModel model, + FeatureConfiguration featureConfiguration) { + CppCompilationContext.Builder contextBuilder = + new CppCompilationContext.Builder(ruleContext); + + // Setup the include path; local include directories come before those inherited from deps or + // from the toolchain; in case of aliasing (same include file found on different entries), + // prefer the local include rather than the inherited one. + + // Add in the roots for well-formed include names for source files and + // generated files. It is important that the execRoot (EMPTY_FRAGMENT) comes + // before the genfilesFragment to preferably pick up source files. Otherwise + // we might pick up stale generated files. + contextBuilder.addQuoteIncludeDir(PathFragment.EMPTY_FRAGMENT); + contextBuilder.addQuoteIncludeDir(ruleContext.getConfiguration().getGenfilesFragment()); + + for (PathFragment systemIncludeDir : systemIncludeDirs) { + contextBuilder.addSystemIncludeDir(systemIncludeDir); + } + for (PathFragment includeDir : includeDirs) { + contextBuilder.addIncludeDir(includeDir); + } + + contextBuilder.mergeDependentContexts( + AnalysisUtils.getProviders(deps, CppCompilationContext.class)); + CppHelper.mergeToolchainDependentContext(ruleContext, contextBuilder); + + // But defines come after those inherited from deps. + contextBuilder.addDefines(defines); + + // There are no ordering constraints for declared include dirs/srcs, or the pregrepped headers. + contextBuilder.addDeclaredIncludeSrcs(publicHeaders); + contextBuilder.addDeclaredIncludeSrcs(privateHeaders); + contextBuilder.addPregreppedHeaderMap( + CppHelper.createExtractInclusions(ruleContext, publicHeaders)); + contextBuilder.addPregreppedHeaderMap( + CppHelper.createExtractInclusions(ruleContext, privateHeaders)); + contextBuilder.addCompilationPrerequisites(prerequisites); + + // Add this package's dir to declaredIncludeDirs, & this rule's headers to declaredIncludeSrcs + // Note: no include dir for STRICT mode. + if (headersCheckingMode == HeadersCheckingMode.WARN) { + contextBuilder.addDeclaredIncludeWarnDir(ruleContext.getLabel().getPackageFragment()); + for (PathFragment looseIncludeDir : looseIncludeDirs) { + contextBuilder.addDeclaredIncludeWarnDir(looseIncludeDir); + } + } else if (headersCheckingMode == HeadersCheckingMode.LOOSE) { + contextBuilder.addDeclaredIncludeDir(ruleContext.getLabel().getPackageFragment()); + for (PathFragment looseIncludeDir : looseIncludeDirs) { + contextBuilder.addDeclaredIncludeDir(looseIncludeDir); + } + } + + if (emitCppModuleMaps) { + CppModuleMap cppModuleMap = CppHelper.addCppModuleMapToContext(ruleContext, contextBuilder); + // TODO(bazel-team): addCppModuleMapToContext second-guesses whether module maps should + // actually be enabled, so we need to double-check here. Who would write code like this? + if (cppModuleMap != null) { + CppModuleMapAction action = new CppModuleMapAction(ruleContext.getActionOwner(), + cppModuleMap, + privateHeaders, + publicHeaders, + collectModuleMaps(), + additionalExportedHeaders, + compileHeaderModules, + featureConfiguration.isEnabled(CppRuleClasses.MODULE_MAP_HOME_CWD)); + ruleContext.registerAction(action); + } + if (model.getGeneratesPicHeaderModule()) { + contextBuilder.setPicHeaderModule(model.getPicHeaderModule(cppModuleMap.getArtifact())); + } + if (model.getGeratesNoPicHeaderModule()) { + contextBuilder.setHeaderModule(model.getHeaderModule(cppModuleMap.getArtifact())); + } + } + + semantics.setupCompilationContext(ruleContext, contextBuilder); + return contextBuilder.build(); + } + + private Iterable<CppModuleMap> collectModuleMaps() { + // Cpp module maps may be null for some rules. We filter the nulls out at the end. + List<CppModuleMap> result = new ArrayList<>(); + Iterables.addAll(result, Iterables.transform(deps, CPP_DEPS_TO_MODULES)); + CppCompilationContext stl = + ruleContext.getPrerequisite(":stl", Mode.TARGET, CppCompilationContext.class); + if (stl != null) { + result.add(stl.getCppModuleMap()); + } + + CcToolchainProvider toolchain = CppHelper.getToolchain(ruleContext); + if (toolchain != null) { + result.add(toolchain.getCppCompilationContext().getCppModuleMap()); + } + + if (emitHeaderTargetModuleMaps) { + for (HeaderTargetModuleMapProvider provider : AnalysisUtils.getProviders( + deps, HeaderTargetModuleMapProvider.class)) { + result.addAll(provider.getCppModuleMaps()); + } + } + + return Iterables.filter(result, Predicates.<CppModuleMap>notNull()); + } + + private TransitiveLipoInfoProvider collectTransitiveLipoInfo(CcCompilationOutputs outputs) { + if (ruleContext.getFragment(CppConfiguration.class).getFdoSupport().getFdoRoot() == null) { + return TransitiveLipoInfoProvider.EMPTY; + } + NestedSetBuilder<IncludeScannable> scannableBuilder = NestedSetBuilder.stableOrder(); + // TODO(bazel-team): Only fetch the STL prerequisite in one place. + TransitiveInfoCollection stl = ruleContext.getPrerequisite(":stl", Mode.TARGET); + if (stl != null) { + TransitiveLipoInfoProvider provider = stl.getProvider(TransitiveLipoInfoProvider.class); + if (provider != null) { + scannableBuilder.addTransitive(provider.getTransitiveIncludeScannables()); + } + } + + for (TransitiveLipoInfoProvider dep : + AnalysisUtils.getProviders(deps, TransitiveLipoInfoProvider.class)) { + scannableBuilder.addTransitive(dep.getTransitiveIncludeScannables()); + } + + for (IncludeScannable scannable : outputs.getLipoScannables()) { + Preconditions.checkState(scannable.getIncludeScannerSources().size() == 1); + scannableBuilder.add(scannable); + } + return new TransitiveLipoInfoProvider(scannableBuilder.build()); + } + + private Runfiles collectCppRunfiles( + CcLinkingOutputs ccLinkingOutputs, boolean linkingStatically) { + Runfiles.Builder builder = new Runfiles.Builder(); + builder.addTargets(deps, RunfilesProvider.DEFAULT_RUNFILES); + builder.addTargets(deps, CppRunfilesProvider.runfilesFunction(linkingStatically)); + // Add the shared libraries to the runfiles. + builder.addArtifacts(ccLinkingOutputs.getLibrariesForRunfiles(linkingStatically)); + return builder.build(); + } + + private CcLinkParamsStore createCcLinkParamsStore( + final CcLinkingOutputs ccLinkingOutputs, final CppCompilationContext cppCompilationContext, + final boolean forcePic) { + return new CcLinkParamsStore() { + @Override + protected void collect(CcLinkParams.Builder builder, boolean linkingStatically, + boolean linkShared) { + builder.addLinkstamps(linkstamps, cppCompilationContext); + builder.addTransitiveTargets(deps, + CcLinkParamsProvider.TO_LINK_PARAMS, CcSpecificLinkParamsProvider.TO_LINK_PARAMS); + if (!neverlink) { + builder.addLibraries(ccLinkingOutputs.getPreferredLibraries(linkingStatically, + /*preferPic=*/linkShared || forcePic)); + builder.addLinkOpts(linkopts); + } + } + }; + } + + private NestedSet<LinkerInput> collectNativeCcLibraries(CcLinkingOutputs ccLinkingOutputs) { + NestedSetBuilder<LinkerInput> result = NestedSetBuilder.linkOrder(); + result.addAll(ccLinkingOutputs.getDynamicLibraries()); + for (CcNativeLibraryProvider dep : AnalysisUtils.getProviders( + deps, CcNativeLibraryProvider.class)) { + result.addTransitive(dep.getTransitiveCcNativeLibraries()); + } + + return result.build(); + } + + private CcExecutionDynamicLibrariesProvider collectExecutionDynamicLibraryArtifacts( + List<LibraryToLink> executionDynamicLibraries) { + Iterable<Artifact> artifacts = LinkerInputs.toLibraryArtifacts(executionDynamicLibraries); + if (!Iterables.isEmpty(artifacts)) { + return new CcExecutionDynamicLibrariesProvider( + NestedSetBuilder.wrap(Order.STABLE_ORDER, artifacts)); + } + + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (CcExecutionDynamicLibrariesProvider dep : + AnalysisUtils.getProviders(deps, CcExecutionDynamicLibrariesProvider.class)) { + builder.addTransitive(dep.getExecutionDynamicLibraryArtifacts()); + } + return builder.isEmpty() + ? CcExecutionDynamicLibrariesProvider.EMPTY + : new CcExecutionDynamicLibrariesProvider(builder.build()); + } + + private TempsProvider getTemps(CcCompilationOutputs compilationOutputs) { + return ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector() + ? new TempsProvider(ImmutableList.<Artifact>of()) + : new TempsProvider(compilationOutputs.getTemps()); + } + + private ImmutableList<Artifact> getFilesToCompile(CcCompilationOutputs compilationOutputs) { + return ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector() + ? ImmutableList.<Artifact>of() + : compilationOutputs.getObjectFiles(CppHelper.usePic(ruleContext, false)); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParams.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParams.java new file mode 100644 index 0000000000..4e4804b177 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParams.java @@ -0,0 +1,357 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; + +import java.util.Collection; +import java.util.Objects; + +/** + * Parameters to be passed to the linker. + * + * <p>The parameters concerned are the link options (strings) passed to the linker, linkstamps and a + * list of libraries to be linked in. + * + * <p>Items in the collections are stored in nested sets. Link options and libraries are stored in + * link order (preorder) and linkstamps are sorted. + */ +public final class CcLinkParams { + private final NestedSet<ImmutableList<String>> linkOpts; + private final NestedSet<Linkstamp> linkstamps; + private final NestedSet<LibraryToLink> libraries; + + private CcLinkParams(NestedSet<ImmutableList<String>> linkOpts, + NestedSet<Linkstamp> linkstamps, + NestedSet<LibraryToLink> libraries) { + this.linkOpts = linkOpts; + this.linkstamps = linkstamps; + this.libraries = libraries; + } + + /** + * @return the linkopts + */ + public NestedSet<ImmutableList<String>> getLinkopts() { + return linkOpts; + } + + public ImmutableList<String> flattenedLinkopts() { + return ImmutableList.copyOf(Iterables.concat(linkOpts)); + } + + /** + * @return the linkstamps + */ + public NestedSet<Linkstamp> getLinkstamps() { + return linkstamps; + } + + /** + * @return the libraries + */ + public NestedSet<LibraryToLink> getLibraries() { + return libraries; + } + + public static final Builder builder(boolean linkingStatically, boolean linkShared) { + return new Builder(linkingStatically, linkShared); + } + + /** + * Builder for {@link CcLinkParams}. + * + * + */ + public static final class Builder { + + /** + * linkingStatically is true when we're linking this target in either FULLY STATIC mode + * (linkopts=["-static"]) or MOSTLY STATIC mode (linkstatic=1). When this is true, we want to + * use static versions of any libraries that this target depends on (except possibly system + * libraries, which are not handled by CcLinkParams). When this is false, we want to use dynamic + * versions of any libraries that this target depends on. + */ + private final boolean linkingStatically; + + /** + * linkShared is true when we're linking with "-shared" (linkshared=1). + */ + private final boolean linkShared; + + private ImmutableList.Builder<String> localLinkoptsBuilder = ImmutableList.builder(); + + private final NestedSetBuilder<ImmutableList<String>> linkOptsBuilder = + NestedSetBuilder.linkOrder(); + private final NestedSetBuilder<Linkstamp> linkstampsBuilder = + NestedSetBuilder.compileOrder(); + private final NestedSetBuilder<LibraryToLink> librariesBuilder = + NestedSetBuilder.linkOrder(); + + private boolean built = false; + + private Builder(boolean linkingStatically, boolean linkShared) { + this.linkingStatically = linkingStatically; + this.linkShared = linkShared; + } + + /** + * Build a {@link CcLinkParams} object. + */ + public CcLinkParams build() { + Preconditions.checkState(!built); + // Not thread-safe, but builders should not be shared across threads. + built = true; + ImmutableList<String> localLinkopts = localLinkoptsBuilder.build(); + if (!localLinkopts.isEmpty()) { + linkOptsBuilder.add(localLinkopts); + } + return new CcLinkParams(linkOptsBuilder.build(), linkstampsBuilder.build(), + librariesBuilder.build()); + } + + private boolean add(CcLinkParamsStore store) { + if (store != null) { + CcLinkParams args = store.get(linkingStatically, linkShared); + addTransitiveArgs(args); + } + return store != null; + } + + /** + * Includes link parameters from a collection of dependency targets. + */ + public Builder addTransitiveTargets(Iterable<? extends TransitiveInfoCollection> targets) { + for (TransitiveInfoCollection target : targets) { + addTransitiveTarget(target); + } + return this; + } + + /** + * Includes link parameters from a dependency target. + * + * <p>The target should implement {@link CcLinkParamsProvider}. If it does not, + * the method does not do anything. + */ + public Builder addTransitiveTarget(TransitiveInfoCollection target) { + return addTransitiveProvider(target.getProvider(CcLinkParamsProvider.class)); + } + + /** + * Includes link parameters from a dependency target. The target is checked for the given + * mappings in the order specified, and the first mapping that returns a non-null result is + * added. + */ + @SafeVarargs + public final Builder addTransitiveTarget(TransitiveInfoCollection target, + Function<TransitiveInfoCollection, CcLinkParamsStore> firstMapping, + @SuppressWarnings("unchecked") // Java arrays don't preserve generic arguments. + Function<TransitiveInfoCollection, CcLinkParamsStore>... remainingMappings) { + if (add(firstMapping.apply(target))) { + return this; + } + for (Function<TransitiveInfoCollection, CcLinkParamsStore> mapping : remainingMappings) { + if (add(mapping.apply(target))) { + return this; + } + } + return this; + } + + /** + * Includes link parameters from a CcLinkParamsProvider provider. + */ + public Builder addTransitiveProvider(CcLinkParamsProvider provider) { + if (provider == null) { + return this; + } + + CcLinkParams args = provider.getCcLinkParams(linkingStatically, linkShared); + addTransitiveArgs(args); + return this; + } + + /** + * Includes link parameters from the given targets. Each target is checked for the given + * mappings in the order specified, and the first mapping that returns a non-null result is + * added. + */ + @SafeVarargs + public final Builder addTransitiveTargets( + Iterable<? extends TransitiveInfoCollection> targets, + Function<TransitiveInfoCollection, CcLinkParamsStore> firstMapping, + @SuppressWarnings("unchecked") // Java arrays don't preserve generic arguments. + Function<TransitiveInfoCollection, CcLinkParamsStore>... remainingMappings) { + for (TransitiveInfoCollection target : targets) { + addTransitiveTarget(target, firstMapping, remainingMappings); + } + return this; + } + + /** + * Includes link parameters from the given targets. Each target is checked for the given + * mappings in the order specified, and the first mapping that returns a non-null result is + * added. + * + * @deprecated don't add any new uses; all existing uses need to be audited and possibly merged + * into a single call - some of them may introduce semantic changes which need to be + * carefully vetted + */ + @Deprecated + @SafeVarargs + public final Builder addTransitiveLangTargets( + Iterable<? extends TransitiveInfoCollection> targets, + Function<TransitiveInfoCollection, CcLinkParamsStore> firstMapping, + @SuppressWarnings("unchecked") // Java arrays don't preserve generic arguments. + Function<TransitiveInfoCollection, CcLinkParamsStore>... remainingMappings) { + return addTransitiveTargets(targets, firstMapping, remainingMappings); + } + + /** + * Merges the other {@link CcLinkParams} object into this one. + */ + public Builder addTransitiveArgs(CcLinkParams args) { + linkOptsBuilder.addTransitive(args.getLinkopts()); + linkstampsBuilder.addTransitive(args.getLinkstamps()); + librariesBuilder.addTransitive(args.getLibraries()); + return this; + } + + /** + * Adds a collection of link options. + */ + public Builder addLinkOpts(Collection<String> linkOpts) { + localLinkoptsBuilder.addAll(linkOpts); + return this; + } + + /** + * Adds a collection of linkstamps. + */ + public Builder addLinkstamps(Iterable<Artifact> linkstamps, CppCompilationContext context) { + ImmutableList<Artifact> declaredIncludeSrcs = + ImmutableList.copyOf(context.getDeclaredIncludeSrcs()); + for (Artifact linkstamp : linkstamps) { + linkstampsBuilder.add(new Linkstamp(linkstamp, declaredIncludeSrcs)); + } + return this; + } + + /** + * Adds a library artifact. + */ + public Builder addLibrary(LibraryToLink library) { + librariesBuilder.add(library); + return this; + } + + /** + * Adds a collection of library artifacts. + */ + public Builder addLibraries(Iterable<LibraryToLink> libraries) { + librariesBuilder.addAll(libraries); + return this; + } + + /** + * Processes typical dependencies a C/C++ library. + * + * <p>A helper method that processes getValues() and merges contents of + * getPreferredLibraries() and getLinkOpts() into the current link params + * object. + */ + public Builder addCcLibrary(RuleContext context, CcCommon common, boolean neverlink, + CcLinkingOutputs linkingOutputs) { + addTransitiveTargets( + context.getPrerequisites("deps", Mode.TARGET), + CcLinkParamsProvider.TO_LINK_PARAMS, CcSpecificLinkParamsProvider.TO_LINK_PARAMS); + + if (!neverlink) { + addLibraries(linkingOutputs.getPreferredLibraries(linkingStatically, + linkShared || context.getFragment(CppConfiguration.class).forcePic())); + addLinkOpts(common.getLinkopts()); + } + return this; + } + } + + /** + * A linkstamp that also knows about its declared includes. + * + * <p>This object is required because linkstamp files may include other headers which + * will have to be provided during compilation. + */ + public static final class Linkstamp { + private final Artifact artifact; + private final ImmutableList<Artifact> declaredIncludeSrcs; + + private Linkstamp(Artifact artifact, ImmutableList<Artifact> declaredIncludeSrcs) { + this.artifact = Preconditions.checkNotNull(artifact); + this.declaredIncludeSrcs = Preconditions.checkNotNull(declaredIncludeSrcs); + } + + /** + * Returns the linkstamp artifact. + */ + public Artifact getArtifact() { + return artifact; + } + + /** + * Returns the declared includes. + */ + public ImmutableList<Artifact> getDeclaredIncludeSrcs() { + return declaredIncludeSrcs; + } + + @Override + public int hashCode() { + return Objects.hash(artifact, declaredIncludeSrcs); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof Linkstamp)) { + return false; + } + Linkstamp other = (Linkstamp) obj; + return artifact.equals(other.artifact) + && declaredIncludeSrcs.equals(other.declaredIncludeSrcs); + } + } + + /** + * Empty CcLinkParams. + */ + public static final CcLinkParams EMPTY = new CcLinkParams( + NestedSetBuilder.<ImmutableList<String>>emptySet(Order.LINK_ORDER), + NestedSetBuilder.<Linkstamp>emptySet(Order.COMPILE_ORDER), + NestedSetBuilder.<LibraryToLink>emptySet(Order.LINK_ORDER)); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsProvider.java new file mode 100644 index 0000000000..11f6011f50 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsProvider.java @@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore.CcLinkParamsStoreImpl; + +/** + * A target that provides C linker parameters. + */ +@Immutable +public final class CcLinkParamsProvider implements TransitiveInfoProvider { + public static final Function<TransitiveInfoCollection, CcLinkParamsStore> TO_LINK_PARAMS = + new Function<TransitiveInfoCollection, CcLinkParamsStore>() { + @Override + public CcLinkParamsStore apply(TransitiveInfoCollection input) { + CcLinkParamsProvider provider = input.getProvider( + CcLinkParamsProvider.class); + return provider == null ? null : provider.store; + } + }; + + private final CcLinkParamsStoreImpl store; + + public CcLinkParamsProvider(CcLinkParamsStore store) { + this.store = new CcLinkParamsStoreImpl(store); + } + + /** + * Returns link parameters given static / shared linking settings. + */ + public CcLinkParams getCcLinkParams(boolean linkingStatically, boolean linkShared) { + return store.get(linkingStatically, linkShared); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsStore.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsStore.java new file mode 100644 index 0000000000..a150488d84 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsStore.java @@ -0,0 +1,136 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.rules.cpp.CcLinkParams.Builder; + +/** + * A cache of C link parameters. + * + * <p>The cache holds instances of {@link CcLinkParams} for combinations of + * linkingStatically and linkShared. If a requested value is not available in + * the cache, it is computed and then stored. + * + * <p>Typically this class is used on targets that may be linked in as C + * libraries as in the following example: + * + * <pre> + * class SomeTarget implements CcLinkParamsProvider { + * private final CcLinkParamsStore ccLinkParamsStore = new CcLinkParamsStore() { + * @Override + * protected void collect(CcLinkParams.Builder builder, boolean linkingStatically, + * boolean linkShared) { + * builder.add[...] + * } + * }; + * + * @Override + * public CcLinkParams getCcLinkParams(boolean linkingStatically, boolean linkShared) { + * return ccLinkParamsStore.get(linkingStatically, linkShared); + * } + * } + * </pre> + */ +public abstract class CcLinkParamsStore { + + private CcLinkParams staticSharedParams; + private CcLinkParams staticNoSharedParams; + private CcLinkParams noStaticSharedParams; + private CcLinkParams noStaticNoSharedParams; + + private CcLinkParams compute(boolean linkingStatically, boolean linkShared) { + CcLinkParams.Builder builder = CcLinkParams.builder(linkingStatically, linkShared); + collect(builder, linkingStatically, linkShared); + return builder.build(); + } + + /** + * Returns {@link CcLinkParams} for a combination of parameters. + * + * <p>The {@link CcLinkParams} instance is computed lazily and cached. + */ + public synchronized CcLinkParams get(boolean linkingStatically, boolean linkShared) { + CcLinkParams result = lookup(linkingStatically, linkShared); + if (result == null) { + result = compute(linkingStatically, linkShared); + put(linkingStatically, linkShared, result); + } + return result; + } + + private CcLinkParams lookup(boolean linkingStatically, boolean linkShared) { + if (linkingStatically) { + return linkShared ? staticSharedParams : staticNoSharedParams; + } else { + return linkShared ? noStaticSharedParams : noStaticNoSharedParams; + } + } + + private void put(boolean linkingStatically, boolean linkShared, CcLinkParams params) { + Preconditions.checkNotNull(params); + if (linkingStatically) { + if (linkShared) { + staticSharedParams = params; + } else { + staticNoSharedParams = params; + } + } else { + if (linkShared) { + noStaticSharedParams = params; + } else { + noStaticNoSharedParams = params; + } + } + } + + /** + * Hook for building the actual link params. + * + * <p>Users should override this method and call methods of the builder to + * set up the actual CcLinkParams objects. + * + * <p>Implementations of this method must not fail or try to report errors on the + * configured target. + */ + protected abstract void collect(CcLinkParams.Builder builder, boolean linkingStatically, + boolean linkShared); + + /** + * An empty CcLinkParamStore. + */ + public static final CcLinkParamsStore EMPTY = new CcLinkParamsStore() { + + @Override + protected void collect(Builder builder, boolean linkingStatically, boolean linkShared) {} + }; + + /** + * An implementation class for the CcLinkParamsStore. + */ + public static final class CcLinkParamsStoreImpl extends CcLinkParamsStore { + + public CcLinkParamsStoreImpl(CcLinkParamsStore store) { + super.staticSharedParams = store.get(true, true); + super.staticNoSharedParams = store.get(true, false); + super.noStaticSharedParams = store.get(false, true); + super.noStaticNoSharedParams = store.get(false, false); + } + + @Override + protected void collect(Builder builder, boolean linkingStatically, boolean linkShared) {} + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkingOutputs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkingOutputs.java new file mode 100644 index 0000000000..6b45c79645 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkingOutputs.java @@ -0,0 +1,243 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.vfs.FileSystemUtils; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * A structured representation of the link outputs of a C++ rule. + */ +public class CcLinkingOutputs { + + public static final CcLinkingOutputs EMPTY = new Builder().build(); + + private final ImmutableList<LibraryToLink> staticLibraries; + + private final ImmutableList<LibraryToLink> picStaticLibraries; + + private final ImmutableList<LibraryToLink> dynamicLibraries; + + private final ImmutableList<LibraryToLink> executionDynamicLibraries; + + private CcLinkingOutputs(ImmutableList<LibraryToLink> staticLibraries, + ImmutableList<LibraryToLink> picStaticLibraries, + ImmutableList<LibraryToLink> dynamicLibraries, + ImmutableList<LibraryToLink> executionDynamicLibraries) { + this.staticLibraries = staticLibraries; + this.picStaticLibraries = picStaticLibraries; + this.dynamicLibraries = dynamicLibraries; + this.executionDynamicLibraries = executionDynamicLibraries; + } + + public ImmutableList<LibraryToLink> getStaticLibraries() { + return staticLibraries; + } + + public ImmutableList<LibraryToLink> getPicStaticLibraries() { + return picStaticLibraries; + } + + public ImmutableList<LibraryToLink> getDynamicLibraries() { + return dynamicLibraries; + } + + public ImmutableList<LibraryToLink> getExecutionDynamicLibraries() { + return executionDynamicLibraries; + } + + /** + * Add the ".a", ".pic.a" and/or ".so" files in appropriate order of preference depending on the + * link preferences. + * + * <p>This method tries to simulate a search path for adding static and dynamic libraries, + * allowing either to be preferred over the other depending on the link {@link LinkStaticness}. + * + * TODO(bazel-team): (2009) we should preserve the relative ordering of first and second + * choice libraries. E.g. if srcs=['foo.a','bar.so','baz.a'] then we should link them in the + * same order. Currently we link entries from the first choice list before those from the + * second choice list, i.e. in the order {@code ['bar.so', 'foo.a', 'baz.a']}. + * + * @param linkingStatically whether to prefer static over dynamic libraries. Should be + * <code>true</code> for binaries that are linked in fully static or mostly static mode. + * @param preferPic whether to prefer pic over non pic libraries (usually used when linking + * shared) + */ + public List<LibraryToLink> getPreferredLibraries( + boolean linkingStatically, boolean preferPic) { + return getPreferredLibraries(linkingStatically, preferPic, false); + } + + /** + * Returns the shared libraries that are linked against and therefore also need to be in the + * runfiles. + */ + public Iterable<Artifact> getLibrariesForRunfiles(boolean linkingStatically) { + List<LibraryToLink> libraries = + getPreferredLibraries(linkingStatically, /*preferPic*/false, true); + return CcCommon.getSharedLibrariesFrom(LinkerInputs.toLibraryArtifacts(libraries)); + } + + /** + * Add the ".a", ".pic.a" and/or ".so" files in appropriate order of + * preference depending on the link preferences. + */ + private List<LibraryToLink> getPreferredLibraries(boolean linkingStatically, boolean preferPic, + boolean forRunfiles) { + List<LibraryToLink> candidates = new ArrayList<>(); + // It's important that this code keeps the invariant that preferPic has no effect on the output + // of .so libraries. That is, the resulting list should contain the same .so files in the same + // order. + if (linkingStatically) { // Prefer the static libraries. + if (preferPic) { + // First choice is the PIC static libraries. + // Second choice is the other static libraries (may cause link error if they're not PIC, + // but I think this is preferable to linking dynamically when you asked for statically). + candidates.addAll(picStaticLibraries); + candidates.addAll(staticLibraries); + } else { + // First choice is the non-pic static libraries (best performance); + // second choice is the staticPicLibraries (at least they're static; + // we can live with the extra overhead of PIC). + candidates.addAll(staticLibraries); + candidates.addAll(picStaticLibraries); + } + candidates.addAll(forRunfiles ? executionDynamicLibraries : dynamicLibraries); + } else { + // First choice is the dynamicLibraries. + candidates.addAll(forRunfiles ? executionDynamicLibraries : dynamicLibraries); + if (preferPic) { + // Second choice is the staticPicLibraries (at least they're PIC, so we won't get a + // link error). + candidates.addAll(picStaticLibraries); + candidates.addAll(staticLibraries); + } else { + candidates.addAll(staticLibraries); + candidates.addAll(picStaticLibraries); + } + } + return filterCandidates(candidates); + } + + /** + * Helper method to filter the candidates by removing equivalent library + * entries from the list of candidates. + * + * @param candidates the library candidates to filter + * @return the list of libraries with equivalent duplicate libraries removed. + */ + private List<LibraryToLink> filterCandidates(List<LibraryToLink> candidates) { + List<LibraryToLink> libraries = new ArrayList<>(); + Set<String> identifiers = new HashSet<>(); + for (LibraryToLink library : candidates) { + if (identifiers.add(libraryIdentifierOf(library.getOriginalLibraryArtifact()))) { + libraries.add(library); + } + } + return libraries; + } + + /** + * Returns the library identifier of an artifact: a string that is different for different + * libraries, but is the same for the shared, static and pic versions of the same library. + */ + private static String libraryIdentifierOf(Artifact libraryArtifact) { + String name = libraryArtifact.getRootRelativePath().getPathString(); + String basename = FileSystemUtils.removeExtension(name); + // Need to special-case file types with double extension. + return name.endsWith(".pic.a") + ? FileSystemUtils.removeExtension(basename) + : name.endsWith(".nopic.a") + ? FileSystemUtils.removeExtension(basename) + : name.endsWith(".pic.lo") + ? FileSystemUtils.removeExtension(basename) + : basename; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private final Set<LibraryToLink> staticLibraries = new LinkedHashSet<>(); + private final Set<LibraryToLink> picStaticLibraries = new LinkedHashSet<>(); + private final Set<LibraryToLink> dynamicLibraries = new LinkedHashSet<>(); + private final Set<LibraryToLink> executionDynamicLibraries = new LinkedHashSet<>(); + + public CcLinkingOutputs build() { + return new CcLinkingOutputs(ImmutableList.copyOf(staticLibraries), + ImmutableList.copyOf(picStaticLibraries), ImmutableList.copyOf(dynamicLibraries), + ImmutableList.copyOf(executionDynamicLibraries)); + } + + public Builder merge(CcLinkingOutputs outputs) { + staticLibraries.addAll(outputs.getStaticLibraries()); + picStaticLibraries.addAll(outputs.getPicStaticLibraries()); + dynamicLibraries.addAll(outputs.getDynamicLibraries()); + executionDynamicLibraries.addAll(outputs.getExecutionDynamicLibraries()); + return this; + } + + public Builder addStaticLibrary(LibraryToLink library) { + staticLibraries.add(library); + return this; + } + + public Builder addStaticLibraries(Iterable<LibraryToLink> libraries) { + Iterables.addAll(staticLibraries, libraries); + return this; + } + + public Builder addPicStaticLibrary(LibraryToLink library) { + picStaticLibraries.add(library); + return this; + } + + public Builder addPicStaticLibraries(Iterable<LibraryToLink> libraries) { + Iterables.addAll(picStaticLibraries, libraries); + return this; + } + + public Builder addDynamicLibrary(LibraryToLink library) { + dynamicLibraries.add(library); + return this; + } + + public Builder addDynamicLibraries(Iterable<LibraryToLink> libraries) { + Iterables.addAll(dynamicLibraries, libraries); + return this; + } + + public Builder addExecutionDynamicLibrary(LibraryToLink library) { + executionDynamicLibraries.add(library); + return this; + } + + public Builder addExecutionDynamicLibraries(Iterable<LibraryToLink> libraries) { + Iterables.addAll(executionDynamicLibraries, libraries); + return this; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcNativeLibraryProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcNativeLibraryProvider.java new file mode 100644 index 0000000000..5e96291520 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcNativeLibraryProvider.java @@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A target that provides native libraries in the transitive closure of its deps that are needed for + * executing C++ code. + */ +@Immutable +public final class CcNativeLibraryProvider implements TransitiveInfoProvider { + + private final NestedSet<LinkerInput> transitiveCcNativeLibraries; + + public CcNativeLibraryProvider(NestedSet<LinkerInput> transitiveCcNativeLibraries) { + this.transitiveCcNativeLibraries = transitiveCcNativeLibraries; + } + + /** + * Collects native libraries in the transitive closure of its deps that are needed for executing + * C/C++ code. + * + * <p>In effect, returns all dynamic library (.so) artifacts provided by the transitive closure. + */ + public NestedSet<LinkerInput> getTransitiveCcNativeLibraries() { + return transitiveCcNativeLibraries; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcSpecificLinkParamsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcSpecificLinkParamsProvider.java new file mode 100644 index 0000000000..dfcecc276c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcSpecificLinkParamsProvider.java @@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore.CcLinkParamsStoreImpl; + +/** + * A target that provides libraries to be only linked into other C++ targets (and not targets + * for other languages) + */ +@Immutable +public final class CcSpecificLinkParamsProvider implements TransitiveInfoProvider { + private final CcLinkParamsStoreImpl store; + + public CcSpecificLinkParamsProvider(CcLinkParamsStore store) { + this.store = new CcLinkParamsStoreImpl(store); + } + + public CcLinkParamsStore getLinkParams() { + return store; + } + + public static final Function<TransitiveInfoCollection, CcLinkParamsStore> TO_LINK_PARAMS = + new Function<TransitiveInfoCollection, CcLinkParamsStore>() { + @Override + public CcLinkParamsStore apply(TransitiveInfoCollection input) { + CcSpecificLinkParamsProvider provider = input.getProvider( + CcSpecificLinkParamsProvider.class); + return provider == null ? null : provider.getLinkParams(); + } + }; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcTest.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcTest.java new file mode 100644 index 0000000000..78271836b0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcTest.java @@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * A configured target class for cc_test rules. + */ +public abstract class CcTest implements RuleConfiguredTargetFactory { + + private final CppSemantics semantics; + + protected CcTest(CppSemantics semantics) { + this.semantics = semantics; + } + + @Override + public ConfiguredTarget create(RuleContext context) throws InterruptedException { + return CcBinary.init(semantics, context, /*fake =*/ false, /*useTestOnlyFlags =*/ true); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java new file mode 100644 index 0000000000..bd39d0f745 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java @@ -0,0 +1,249 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Actions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.CompilationHelper; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.LicensesProvider; +import com.google.devtools.build.lib.analysis.LicensesProvider.TargetLicense; +import com.google.devtools.build.lib.analysis.MiddlemanProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.License; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.List; + +/** + * Implementation for the cc_toolchain rule. + */ +public class CcToolchain implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + final Label label = ruleContext.getLabel(); + final NestedSet<Artifact> crosstool = ruleContext.getPrerequisite("all_files", Mode.HOST) + .getProvider(FileProvider.class).getFilesToBuild(); + final NestedSet<Artifact> crosstoolMiddleman = getFiles(ruleContext, "all_files"); + final NestedSet<Artifact> compile = getFiles(ruleContext, "compiler_files"); + final NestedSet<Artifact> strip = getFiles(ruleContext, "strip_files"); + final NestedSet<Artifact> objcopy = getFiles(ruleContext, "objcopy_files"); + final NestedSet<Artifact> link = getFiles(ruleContext, "linker_files"); + final NestedSet<Artifact> dwp = getFiles(ruleContext, "dwp_files"); + final NestedSet<Artifact> libcLink = inputsForLibcLink(ruleContext); + String purposePrefix = Actions.escapeLabel(label) + "_"; + String runtimeSolibDirBase = "_solib_" + "_" + Actions.escapeLabel(label); + final PathFragment runtimeSolibDir = ruleContext.getConfiguration() + .getBinFragment().getRelative(runtimeSolibDirBase); + + CppConfiguration cppConfiguration = ruleContext.getFragment(CppConfiguration.class); + // Static runtime inputs. + TransitiveInfoCollection staticRuntimeLibDep = selectDep(ruleContext, "static_runtime_libs", + cppConfiguration.getStaticRuntimeLibsLabel()); + final NestedSet<Artifact> staticRuntimeLinkInputs; + final Artifact staticRuntimeLinkMiddleman; + if (cppConfiguration.supportsEmbeddedRuntimes()) { + staticRuntimeLinkInputs = staticRuntimeLibDep + .getProvider(FileProvider.class) + .getFilesToBuild(); + } else { + staticRuntimeLinkInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (!staticRuntimeLinkInputs.isEmpty()) { + NestedSet<Artifact> staticRuntimeLinkMiddlemanSet = CompilationHelper.getAggregatingMiddleman( + ruleContext, + purposePrefix + "static_runtime_link", + staticRuntimeLibDep); + staticRuntimeLinkMiddleman = staticRuntimeLinkMiddlemanSet.isEmpty() + ? null : Iterables.getOnlyElement(staticRuntimeLinkMiddlemanSet); + } else { + staticRuntimeLinkMiddleman = null; + } + + Preconditions.checkState( + (staticRuntimeLinkMiddleman == null) == staticRuntimeLinkInputs.isEmpty()); + + // Dynamic runtime inputs. + TransitiveInfoCollection dynamicRuntimeLibDep = selectDep(ruleContext, "dynamic_runtime_libs", + cppConfiguration.getDynamicRuntimeLibsLabel()); + final NestedSet<Artifact> dynamicRuntimeLinkInputs; + final Artifact dynamicRuntimeLinkMiddleman; + if (cppConfiguration.supportsEmbeddedRuntimes()) { + NestedSetBuilder<Artifact> dynamicRuntimeLinkInputsBuilder = NestedSetBuilder.stableOrder(); + for (Artifact artifact : dynamicRuntimeLibDep + .getProvider(FileProvider.class).getFilesToBuild()) { + if (CppHelper.SHARED_LIBRARY_FILETYPES.matches(artifact.getFilename())) { + dynamicRuntimeLinkInputsBuilder.add(SolibSymlinkAction.getCppRuntimeSymlink( + ruleContext, artifact, runtimeSolibDirBase, + ruleContext.getConfiguration()).getArtifact()); + } else { + dynamicRuntimeLinkInputsBuilder.add(artifact); + } + } + dynamicRuntimeLinkInputs = dynamicRuntimeLinkInputsBuilder.build(); + } else { + dynamicRuntimeLinkInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (!dynamicRuntimeLinkInputs.isEmpty()) { + List<Artifact> dynamicRuntimeLinkMiddlemanSet = + CppHelper.getAggregatingMiddlemanForCppRuntimes( + ruleContext, + purposePrefix + "dynamic_runtime_link", + dynamicRuntimeLibDep, + runtimeSolibDirBase, + ruleContext.getConfiguration()); + dynamicRuntimeLinkMiddleman = dynamicRuntimeLinkMiddlemanSet.isEmpty() + ? null : Iterables.getOnlyElement(dynamicRuntimeLinkMiddlemanSet); + } else { + dynamicRuntimeLinkMiddleman = null; + } + + Preconditions.checkState( + (dynamicRuntimeLinkMiddleman == null) == dynamicRuntimeLinkInputs.isEmpty()); + + CppCompilationContext.Builder contextBuilder = + new CppCompilationContext.Builder(ruleContext); + CppModuleMap moduleMap = createCrosstoolModuleMap(ruleContext); + if (moduleMap != null) { + contextBuilder.setCppModuleMap(moduleMap); + } + final CppCompilationContext context = contextBuilder.build(); + boolean supportsParamFiles = ruleContext.attributes().get("supports_param_files", BOOLEAN); + boolean supportsHeaderParsing = + ruleContext.attributes().get("supports_header_parsing", BOOLEAN); + + CcToolchainProvider provider = new CcToolchainProvider( + Preconditions.checkNotNull(ruleContext.getFragment(CppConfiguration.class)), + crosstool, + fullInputsForCrosstool(ruleContext, crosstoolMiddleman), + compile, + strip, + objcopy, + fullInputsForLink(ruleContext, link), + dwp, + libcLink, + staticRuntimeLinkInputs, + staticRuntimeLinkMiddleman, + dynamicRuntimeLinkInputs, + dynamicRuntimeLinkMiddleman, + runtimeSolibDir, + context, + supportsParamFiles, + supportsHeaderParsing); + RuleConfiguredTargetBuilder builder = + new RuleConfiguredTargetBuilder(ruleContext) + .add(CcToolchainProvider.class, provider) + .setFilesToBuild(new NestedSetBuilder<Artifact>(Order.STABLE_ORDER).build()) + .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY)); + + // If output_license is specified on the cc_toolchain rule, override the transitive licenses + // with that one. This is necessary because cc_toolchain is used in the target configuration, + // but it is sort-of-kind-of a tool, but various parts of it are linked into the output... + // ...so we trust the judgment of the author of the cc_toolchain rule to figure out what + // licenses should be propagated to C++ targets. + License outputLicense = ruleContext.getRule().getToolOutputLicense(ruleContext.attributes()); + if (outputLicense != null && outputLicense != License.NO_LICENSE) { + final NestedSet<TargetLicense> license = NestedSetBuilder.create(Order.STABLE_ORDER, + new TargetLicense(ruleContext.getLabel(), outputLicense)); + LicensesProvider licensesProvider = new LicensesProvider() { + @Override + public NestedSet<TargetLicense> getTransitiveLicenses() { + return license; + } + }; + + builder.add(LicensesProvider.class, licensesProvider); + } + + return builder.build(); + } + + private NestedSet<Artifact> inputsForLibcLink(RuleContext ruleContext) { + TransitiveInfoCollection libcLink = ruleContext.getPrerequisite(":libc_link", Mode.HOST); + return libcLink != null + ? libcLink.getProvider(FileProvider.class).getFilesToBuild() + : NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER); + } + + private NestedSet<Artifact> fullInputsForCrosstool(RuleContext ruleContext, + NestedSet<Artifact> crosstoolMiddleman) { + return NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(crosstoolMiddleman) + // Use "libc_link" here, because it is functionally identical to the case + // below. If we introduce separate filegroups for compiling and linking, we + // need to fix that here. + .addTransitive(AnalysisUtils.getMiddlemanFor(ruleContext, ":libc_link")) + .build(); + } + + private NestedSet<Artifact> fullInputsForLink(RuleContext ruleContext, NestedSet<Artifact> link) { + return NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(link) + .addTransitive(AnalysisUtils.getMiddlemanFor(ruleContext, ":libc_link")) + .add(ruleContext.getAnalysisEnvironment().getEmbeddedToolArtifact( + CppRuleClasses.BUILD_INTERFACE_SO)) + .build(); + } + + private CppModuleMap createCrosstoolModuleMap(RuleContext ruleContext) { + if (ruleContext.getPrerequisite("module_map", Mode.HOST) == null) { + return null; + } + Artifact moduleMapArtifact = ruleContext.getPrerequisiteArtifact("module_map", Mode.HOST); + if (moduleMapArtifact == null) { + return null; + } + return new CppModuleMap(moduleMapArtifact, "crosstool"); + } + + private TransitiveInfoCollection selectDep( + RuleContext ruleContext, String attribute, Label label) { + for (TransitiveInfoCollection dep : ruleContext.getPrerequisites(attribute, Mode.TARGET)) { + if (dep.getLabel().equals(label)) { + return dep; + } + } + + return ruleContext.getPrerequisites(attribute, Mode.TARGET).get(0); + } + + private NestedSet<Artifact> getFiles(RuleContext context, String attribute) { + TransitiveInfoCollection dep = context.getPrerequisite(attribute, Mode.HOST); + MiddlemanProvider middlemanProvider = dep.getProvider(MiddlemanProvider.class); + // We use the middleman if we can (if the dep is a filegroup), otherwise, just the regular + // filesToBuild (e.g. if it is a simple input file) + return middlemanProvider != null + ? middlemanProvider.getMiddlemanArtifact() + : dep.getProvider(FileProvider.class).getFilesToBuild(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java new file mode 100644 index 0000000000..29ab45cf4a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java @@ -0,0 +1,802 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.CToolchain; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.Serializable; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; + +/** + * Provides access to features supported by a specific toolchain. + * + * <p>This class can be generated from the CToolchain protocol buffer. + * + * <p>TODO(bazel-team): Implement support for specifying the toolchain configuration directly from + * the BUILD file. + * + * <p>TODO(bazel-team): Find a place to put the public-facing documentation and link to it from + * here. + * + * <p>TODO(bazel-team): Split out Feature as CcToolchainFeature, which will modularize the + * crosstool configuration into one part that is about handling a set of features (including feature + * selection) and one part that is about how to apply a single feature (parsing flags and expanding + * them from build variables). + */ +@Immutable +public class CcToolchainFeatures implements Serializable { + + /** + * Thrown when a flag value cannot be expanded under a set of build variables. + * + * <p>This happens for example when a flag references a variable that is not provided by the + * action, or when a flag group references multiple variables of sequence type. + */ + public static class ExpansionException extends RuntimeException { + ExpansionException(String message) { + super(message); + } + } + + /** + * A piece of a single flag. + * + * <p>A single flag can contain a combination of text and variables (for example + * "-f %{var1}/%{var2}"). We split the flag into chunks, where each chunk represents either a + * text snippet, or a variable that is to be replaced. + */ + interface FlagChunk { + + /** + * Expands this chunk. + * + * @param variables variable names mapped to their values for a single flag expansion. + * @param flag the flag content to append to. + */ + void expand(Map<String, String> variables, StringBuilder flag); + } + + /** + * A plain text chunk of a flag. + */ + @Immutable + private static class StringChunk implements FlagChunk, Serializable { + private final String text; + + private StringChunk(String text) { + this.text = text; + } + + @Override + public void expand(Map<String, String> variables, StringBuilder flag) { + flag.append(text); + } + } + + /** + * A chunk of a flag into which a variable should be expanded. + */ + @Immutable + private static class VariableChunk implements FlagChunk, Serializable { + private final String variableName; + + private VariableChunk(String variableName) { + this.variableName = variableName; + } + + @Override + public void expand(Map<String, String> variables, StringBuilder flag) { + String value = variables.get(variableName); + if (value == null) { + // We check all variables in FlagGroup.expandCommandLine, so if we arrive here with a + // null value, the variable map originally handed to the feature selection must have + // contained an explicit null value. + throw new ExpansionException("Internal blaze error: build variable was set to 'null'."); + } + flag.append(variables.get(variableName)); + } + } + + /** + * Parser for toolchain flags. + * + * <p>A flag contains a snippet of text supporting variable expansion. For example, a flag value + * "-f %{var1}/%{var2}" will expand the values of the variables "var1" and "var2" in the + * corresponding places in the string. + * + * <p>The {@code FlagParser} takes a flag string and parses it into a list of {@code FlagChunk} + * objects, where each chunk represents either a snippet of text or a variable to be expanded. In + * the above example, the resulting chunks would be ["-f ", var1, "/", var2]. + * + * <p>In addition to the list of chunks, the {@code FlagParser} also provides the set of variables + * necessary for the expansion of this flag via {@code getUsedVariables}. + * + * <p>To get a literal percent character, "%%" can be used in the flag text. + */ + private static class FlagParser { + + /** + * The given flag value. + */ + private final String value; + + /** + * The current position in {@value} during parsing. + */ + private int current = 0; + + private final ImmutableList.Builder<FlagChunk> chunks = ImmutableList.builder(); + private final ImmutableSet.Builder<String> usedVariables = ImmutableSet.builder(); + + private FlagParser(String value) throws InvalidConfigurationException { + this.value = value; + parse(); + } + + /** + * @return the parsed chunks for this flag. + */ + private ImmutableList<FlagChunk> getChunks() { + return chunks.build(); + } + + /** + * @return all variable names needed to expand this flag. + */ + private ImmutableSet<String> getUsedVariables() { + return usedVariables.build(); + } + + /** + * Parses the flag. + * + * @throws InvalidConfigurationException if there is a parsing error. + */ + private void parse() throws InvalidConfigurationException { + while (current < value.length()) { + if (atVariableStart()) { + parseVariableChunk(); + } else { + parseStringChunk(); + } + } + } + + /** + * @return whether the current position is the start of a variable. + */ + private boolean atVariableStart() { + // We parse a variable when value starts with '%', but not '%%'. + return value.charAt(current) == '%' + && (current + 1 >= value.length() || value.charAt(current + 1) != '%'); + } + + /** + * Parses a chunk of text until the next '%', which indicates either an escaped literal '%' + * or a variable. + */ + private void parseStringChunk() { + int start = current; + // We only parse string chunks starting with '%' if they also start with '%%'. + // In that case, we want to have a single '%' in the string, so we start at the second + // character. + // Note that for flags like "abc%%def" this will lead to two string chunks, the first + // referencing the subtring "abc", and a second referencing the substring "%def". + if (value.charAt(current) == '%') { + current = current + 1; + start = current; + } + current = value.indexOf('%', current + 1); + if (current == -1) { + current = value.length(); + } + final String text = value.substring(start, current); + chunks.add(new StringChunk(text)); + } + + /** + * Parses a variable to be expanded. + * + * @throws InvalidConfigurationException if there is a parsing error. + */ + private void parseVariableChunk() throws InvalidConfigurationException { + current = current + 1; + if (current >= value.length() || value.charAt(current) != '{') { + abort("expected '{'"); + } + current = current + 1; + if (current >= value.length() || value.charAt(current) == '}') { + abort("expected variable name"); + } + int end = value.indexOf('}', current); + final String name = value.substring(current, end); + usedVariables.add(name); + chunks.add(new VariableChunk(name)); + current = end + 1; + } + + /** + * @throws InvalidConfigurationException with the given error text, adding information about + * the current position in the flag. + */ + private void abort(String error) throws InvalidConfigurationException { + throw new InvalidConfigurationException("Invalid toolchain configuration: " + error + + " at position " + current + " while parsing a flag containing '" + value + "'"); + } + } + + /** + * A single flag to be expanded under a set of variables. + * + * <p>TODO(bazel-team): Consider specializing Flag for the simple case that a flag is just a bit + * of text. + */ + @Immutable + private static class Flag implements Serializable { + private final ImmutableList<FlagChunk> chunks; + + private Flag(ImmutableList<FlagChunk> chunks) { + this.chunks = chunks; + } + + /** + * Expand this flag into a single new entry in {@code commandLine}. + */ + private void expandCommandLine(Map<String, String> variables, List<String> commandLine) { + StringBuilder flag = new StringBuilder(); + for (FlagChunk chunk : chunks) { + chunk.expand(variables, flag); + } + commandLine.add(flag.toString()); + } + } + + /** + * A group of flags. + */ + @Immutable + private static class FlagGroup implements Serializable { + private final ImmutableList<Flag> flags; + private final ImmutableSet<String> usedVariables; + + private FlagGroup(CToolchain.FlagGroup flagGroup) throws InvalidConfigurationException { + ImmutableList.Builder<Flag> flags = ImmutableList.builder(); + ImmutableSet.Builder<String> usedVariables = ImmutableSet.builder(); + for (String flag : flagGroup.getFlagList()) { + FlagParser parser = new FlagParser(flag); + flags.add(new Flag(parser.getChunks())); + usedVariables.addAll(parser.getUsedVariables()); + } + this.flags = flags.build(); + this.usedVariables = usedVariables.build(); + } + + /** + * Expands all flags in this group and adds them to {@code commandLine}. + * + * <p>The flags of the group will be expanded either: + * <ul> + * <li>once, if there is no variable of sequence type in any of the group's flags, or</li> + * <li>for each element in the sequence, if there is one variable of sequence type within + * the flags.</li> + * </ul> + * + * <p>Having more than a single variable of sequence type in a single flag group is not + * supported. + */ + private void expandCommandLine(Multimap<String, String> variables, List<String> commandLine) { + Map<String, String> variableView = new HashMap<>(); + String sequenceName = null; + for (String name : usedVariables) { + Collection<String> value = variables.get(name); + if (value.isEmpty()) { + throw new ExpansionException("Invalid toolchain configuration: unknown variable '" + name + + "' can not be expanded."); + } else if (value.size() > 1) { + if (sequenceName != null) { + throw new ExpansionException( + "Invalid toolchain configuration: trying to expand two variable list in one " + + "flag group: '" + sequenceName + "' and '" + name + "'"); + } + sequenceName = name; + } else { + variableView.put(name, value.iterator().next()); + } + } + if (sequenceName != null) { + for (String value : variables.get(sequenceName)) { + variableView.put(sequenceName, value); + expandOnce(variableView, commandLine); + } + } else { + expandOnce(variableView, commandLine); + } + } + + /** + * Expanding all flags of this group into {@code commandLine}. + */ + private void expandOnce(Map<String, String> variables, List<String> commandLine) { + for (Flag flag : flags) { + flag.expandCommandLine(variables, commandLine); + } + } + } + + /** + * Groups a set of flags to apply for certain actions. + */ + @Immutable + private static class FlagSet implements Serializable { + private final ImmutableSet<String> actions; + private final ImmutableList<FlagGroup> flagGroups; + + private FlagSet(CToolchain.FlagSet flagSet) throws InvalidConfigurationException { + this.actions = ImmutableSet.copyOf(flagSet.getActionList()); + ImmutableList.Builder<FlagGroup> builder = ImmutableList.builder(); + for (CToolchain.FlagGroup flagGroup : flagSet.getFlagGroupList()) { + builder.add(new FlagGroup(flagGroup)); + } + this.flagGroups = builder.build(); + } + + /** + * Adds the flags that apply to the given {@code action} to {@code commandLine}. + */ + private void expandCommandLine(String action, Multimap<String, String> variables, + List<String> commandLine) { + if (!actions.contains(action)) { + return; + } + for (FlagGroup flagGroup : flagGroups) { + flagGroup.expandCommandLine(variables, commandLine); + } + } + } + + /** + * Contains flags for a specific feature. + */ + @Immutable + private static class Feature implements Serializable { + private final String name; + private final ImmutableList<FlagSet> flagSets; + + private Feature(CToolchain.Feature feature) throws InvalidConfigurationException { + this.name = feature.getName(); + ImmutableList.Builder<FlagSet> builder = ImmutableList.builder(); + for (CToolchain.FlagSet flagSet : feature.getFlagSetList()) { + builder.add(new FlagSet(flagSet)); + } + this.flagSets = builder.build(); + } + + /** + * @return the features's name. + */ + private String getName() { + return name; + } + + /** + * Adds the flags that apply to the given {@code action} to {@code commandLine}. + */ + private void expandCommandLine(String action, Multimap<String, String> variables, + List<String> commandLine) { + for (FlagSet flagSet : flagSets) { + flagSet.expandCommandLine(action, variables, commandLine); + } + } + } + + /** + * Captures the set of enabled features for a rule. + */ + @Immutable + public static class FeatureConfiguration { + private final ImmutableSet<String> enabledFeatureNames; + private final ImmutableList<Feature> enabledFeatures; + + public FeatureConfiguration() { + enabledFeatureNames = ImmutableSet.of(); + enabledFeatures = ImmutableList.of(); + } + + private FeatureConfiguration(ImmutableList<Feature> enabledFeatures) { + this.enabledFeatures = enabledFeatures; + ImmutableSet.Builder<String> builder = ImmutableSet.builder(); + for (Feature feature : enabledFeatures) { + builder.add(feature.getName()); + } + this.enabledFeatureNames = builder.build(); + } + + /** + * @return whether the given {@code feature} is enabled. + */ + boolean isEnabled(String feature) { + return enabledFeatureNames.contains(feature); + } + + /** + * @return the command line for the given {@code action}. + */ + List<String> getCommandLine(String action, Multimap<String, String> variables) { + List<String> commandLine = new ArrayList<>(); + for (Feature feature : enabledFeatures) { + feature.expandCommandLine(action, variables, commandLine); + } + return commandLine; + } + } + + /** + * All features in the order in which they were specified in the configuration. + * + * <p>We guarantee the command line to be in the order in which the flags were specified in the + * configuration. + */ + private final ImmutableList<Feature> features; + + /** + * Maps from the feature's name to the feature. + */ + private final ImmutableMap<String, Feature> featuresByName; + + /** + * Maps from a feature to a set of all the features it has a direct 'implies' edge to. + */ + private final ImmutableMultimap<Feature, Feature> implies; + + /** + * Maps from a feature to all features that have an direct 'implies' edge to this feature. + */ + private final ImmutableMultimap<Feature, Feature> impliedBy; + + /** + * Maps from a feature to a set of feature sets, where: + * <ul> + * <li>a feature set satisfies the 'requires' condition, if all features in the feature set are + * enabled</li> + * <li>the 'requires' condition is satisfied, if at least one of the feature sets satisfies the + * 'requires' condition.</li> + * </ul> + */ + private final ImmutableMultimap<Feature, ImmutableSet<Feature>> requires; + + /** + * Maps from a feature to all features that have a requirement referencing it. + * + * <p>This will be used to determine which features need to be re-checked after a feature was + * disabled. + */ + private final ImmutableMultimap<Feature, Feature> requiredBy; + + /** + * A cache of feature selection results, so we do not recalculate the feature selection for + * all actions. + */ + private transient LoadingCache<Collection<String>, FeatureConfiguration> + configurationCache = buildConfigurationCache(); + + /** + * Constructs the feature configuration from a {@code CToolchain} protocol buffer. + * + * @param toolchain the toolchain configuration as specified by the user. + * @throws InvalidConfigurationException if the configuration has logical errors. + */ + CcToolchainFeatures(CToolchain toolchain) throws InvalidConfigurationException { + // Build up the feature graph. + // First, we build up the map of name -> features in one pass, so that earlier features can + // reference later features in their configuration. + ImmutableList.Builder<Feature> features = ImmutableList.builder(); + HashMap<String, Feature> featuresByName = new HashMap<>(); + for (CToolchain.Feature toolchainFeature : toolchain.getFeatureList()) { + Feature feature = new Feature(toolchainFeature); + features.add(feature); + if (featuresByName.put(feature.getName(), feature) != null) { + throw new InvalidConfigurationException("Invalid toolchain configuration: feature '" + + feature.getName() + "' was specified multiple times."); + } + } + this.features = features.build(); + this.featuresByName = ImmutableMap.copyOf(featuresByName); + + // Next, we build up all forward references for 'implies' and 'requires' edges. + ImmutableMultimap.Builder<Feature, Feature> implies = ImmutableMultimap.builder(); + ImmutableMultimap.Builder<Feature, ImmutableSet<Feature>> requires = + ImmutableMultimap.builder(); + // We also store the reverse 'implied by' and 'required by' edges during this pass. + ImmutableMultimap.Builder<Feature, Feature> impliedBy = ImmutableMultimap.builder(); + ImmutableMultimap.Builder<Feature, Feature> requiredBy = ImmutableMultimap.builder(); + for (CToolchain.Feature toolchainFeature : toolchain.getFeatureList()) { + String name = toolchainFeature.getName(); + Feature feature = featuresByName.get(name); + for (CToolchain.FeatureSet requiredFeatures : toolchainFeature.getRequiresList()) { + ImmutableSet.Builder<Feature> allOf = ImmutableSet.builder(); + for (String requiredName : requiredFeatures.getFeatureList()) { + Feature required = getFeatureOrFail(requiredName, name); + allOf.add(required); + requiredBy.put(required, feature); + } + requires.put(feature, allOf.build()); + } + for (String impliedName : toolchainFeature.getImpliesList()) { + Feature implied = getFeatureOrFail(impliedName, name); + impliedBy.put(implied, feature); + implies.put(feature, implied); + } + } + this.implies = implies.build(); + this.requires = requires.build(); + this.impliedBy = impliedBy.build(); + this.requiredBy = requiredBy.build(); + } + + /** + * Assign an empty cache after default-deserializing all non-transient members. + */ + private void readObject(ObjectInputStream in) throws ClassNotFoundException, IOException { + in.defaultReadObject(); + this.configurationCache = buildConfigurationCache(); + } + + /** + * @return an empty {@code FeatureConfiguration} cache. + */ + private LoadingCache<Collection<String>, FeatureConfiguration> buildConfigurationCache() { + return CacheBuilder.newBuilder() + // TODO(klimek): Benchmark and tweak once we support a larger configuration. + .maximumSize(10000) + .build(new CacheLoader<Collection<String>, FeatureConfiguration>() { + @Override + public FeatureConfiguration load(Collection<String> requestedFeatures) { + return computeFeatureConfiguration(requestedFeatures); + } + }); + } + + /** + * Given a list of {@code requestedFeatures}, returns all features that are enabled by the + * toolchain configuration. + * + * <p>A requested feature will not be enabled if the toolchain does not support it (which may + * depend on other requested features). + * + * <p>Additional features will be enabled if the toolchain supports them and they are implied by + * requested features. + */ + FeatureConfiguration getFeatureConfiguration(Collection<String> requestedFeatures) { + return configurationCache.getUnchecked(requestedFeatures); + } + + private FeatureConfiguration computeFeatureConfiguration(Collection<String> requestedFeatures) { + // Command line flags will be output in the order in which they are specified in the toolchain + // configuration. + return new FeatureSelection(requestedFeatures).run(); + } + + /** + * Convenience method taking a variadic string argument list for testing. + */ + FeatureConfiguration getFeatureConfiguration(String... requestedFeatures) { + return getFeatureConfiguration(Arrays.asList(requestedFeatures)); + } + + /** + * @return the feature with the given {@code name}. + * + * @throws InvalidConfigurationException if no feature with the given name was configured. + */ + private Feature getFeatureOrFail(String name, String reference) + throws InvalidConfigurationException { + if (!featuresByName.containsKey(name)) { + throw new InvalidConfigurationException("Invalid toolchain configuration: feature '" + name + + "', which is referenced from feature '" + reference + "', is not defined."); + } + return featuresByName.get(name); + } + + @VisibleForTesting + Collection<String> getFeatureNames() { + Collection<String> featureNames = new HashSet<>(); + for (Feature feature : features) { + featureNames.add(feature.getName()); + } + return featureNames; + } + + /** + * Implements the feature selection algorithm. + * + * <p>Feature selection is done by first enabling all features reachable by an 'implies' edge, + * and then iteratively pruning features that have unmet requirements. + */ + private class FeatureSelection { + + /** + * The features Bazel would like to enable; either because they are supported and generally + * useful, or because the user required them (for example through the command line). + */ + private final ImmutableSet<Feature> requestedFeatures; + + /** + * The currently enabled feature; during feature selection, we first put all features reachable + * via an 'implies' edge into the enabled feature set, and than prune that set from features + * that have unmet requirements. + */ + private Set<Feature> enabled = new HashSet<>(); + + private FeatureSelection(Collection<String> requestedFeatures) { + ImmutableSet.Builder<Feature> builder = ImmutableSet.builder(); + for (String name : requestedFeatures) { + if (featuresByName.containsKey(name)) { + builder.add(featuresByName.get(name)); + } + } + this.requestedFeatures = builder.build(); + } + + /** + * @return all enabled features in the order in which they were specified in the configuration. + */ + private FeatureConfiguration run() { + for (Feature feature : requestedFeatures) { + enableAllImpliedBy(feature); + } + disableUnsupportedFeatures(); + ImmutableList.Builder<Feature> enabledFeaturesInOrder = ImmutableList.builder(); + for (Feature feature : features) { + if (enabled.contains(feature)) { + enabledFeaturesInOrder.add(feature); + } + } + return new FeatureConfiguration(enabledFeaturesInOrder.build()); + } + + /** + * Transitively and unconditionally enable all features implied by the given feature and the + * feature itself to the enabled feature set. + */ + private void enableAllImpliedBy(Feature feature) { + if (enabled.contains(feature)) { + return; + } + enabled.add(feature); + for (Feature implied : implies.get(feature)) { + enableAllImpliedBy(implied); + } + } + + /** + * Remove all unsupported features from the enabled feature set. + */ + private void disableUnsupportedFeatures() { + Queue<Feature> check = new ArrayDeque<>(enabled); + while (!check.isEmpty()) { + checkFeature(check.poll()); + } + } + + /** + * Check if the given feature is still satisfied within the set of currently enabled features. + * + * <p>If it is not, remove the feature from the set of enabled features, and re-check all + * features that may now also become disabled. + */ + private void checkFeature(Feature feature) { + if (!enabled.contains(feature) || isSatisfied(feature)) { + return; + } + enabled.remove(feature); + + // Once we disable a feature, we have to re-check all features that can be affected by + // that removal. + // 1. A feature that implied the current feature is now going to be disabled. + for (Feature impliesCurrent : impliedBy.get(feature)) { + checkFeature(impliesCurrent); + } + // 2. A feature that required the current feature may now be disabled, depending on whether + // the requirement was optional. + for (Feature requiresCurrent : requiredBy.get(feature)) { + checkFeature(requiresCurrent); + } + // 3. A feature that this feature implied may now be disabled if no other feature also implies + // it. + for (Feature implied : implies.get(feature)) { + checkFeature(implied); + } + } + + /** + * @return whether all requirements of the feature are met in the set of currently enabled + * features. + */ + private boolean isSatisfied(Feature feature) { + return (requestedFeatures.contains(feature) || isImpliedByEnabledFeature(feature)) + && allImplicationsEnabled(feature) && allRequirementsMet(feature); + } + + /** + * @return whether a currently enabled feature implies the given feature. + */ + private boolean isImpliedByEnabledFeature(Feature feature) { + for (Feature implies : impliedBy.get(feature)) { + if (enabled.contains(implies)) { + return true; + } + } + return false; + } + + /** + * @return whether all implications of the given feature are enabled. + */ + private boolean allImplicationsEnabled(Feature feature) { + for (Feature implied : implies.get(feature)) { + if (!enabled.contains(implied)) { + return false; + } + } + return true; + } + + /** + * @return whether all requirements are enabled. + * + * <p>This implies that for any of the feature sets all of the specified features are enabled. + */ + private boolean allRequirementsMet(Feature feature) { + if (!requires.containsKey(feature)) { + return true; + } + for (ImmutableSet<Feature> requiresAllOf : requires.get(feature)) { + boolean requirementMet = true; + for (Feature required : requiresAllOf) { + if (!enabled.contains(required)) { + requirementMet = false; + break; + } + } + if (requirementMet) { + return true; + } + } + return false; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java new file mode 100644 index 0000000000..e1940a5d1f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java @@ -0,0 +1,226 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.vfs.PathFragment; + +import javax.annotation.Nullable; + +/** + * Information about a C++ compiler used by the <code>cc_*</code> rules. + */ +@Immutable +public final class CcToolchainProvider implements TransitiveInfoProvider { + /** + * An empty toolchain to be returned in the error case (instead of null). + */ + public static final CcToolchainProvider EMPTY_TOOLCHAIN_IS_ERROR = new CcToolchainProvider( + null, + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + null, + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + null, + PathFragment.EMPTY_FRAGMENT, + CppCompilationContext.EMPTY, + false, + false); + + @Nullable private final CppConfiguration cppConfiguration; + private final NestedSet<Artifact> crosstool; + private final NestedSet<Artifact> crosstoolMiddleman; + private final NestedSet<Artifact> compile; + private final NestedSet<Artifact> strip; + private final NestedSet<Artifact> objCopy; + private final NestedSet<Artifact> link; + private final NestedSet<Artifact> dwp; + private final NestedSet<Artifact> libcLink; + private final NestedSet<Artifact> staticRuntimeLinkInputs; + @Nullable private final Artifact staticRuntimeLinkMiddleman; + private final NestedSet<Artifact> dynamicRuntimeLinkInputs; + @Nullable private final Artifact dynamicRuntimeLinkMiddleman; + private final PathFragment dynamicRuntimeSolibDir; + private final CppCompilationContext cppCompilationContext; + private final boolean supportsParamFiles; + private final boolean supportsHeaderParsing; + + public CcToolchainProvider( + @Nullable CppConfiguration cppConfiguration, + NestedSet<Artifact> crosstool, + NestedSet<Artifact> crosstoolMiddleman, + NestedSet<Artifact> compile, + NestedSet<Artifact> strip, + NestedSet<Artifact> objCopy, + NestedSet<Artifact> link, + NestedSet<Artifact> dwp, + NestedSet<Artifact> libcLink, + NestedSet<Artifact> staticRuntimeLinkInputs, + @Nullable Artifact staticRuntimeLinkMiddleman, + NestedSet<Artifact> dynamicRuntimeLinkInputs, + @Nullable Artifact dynamicRuntimeLinkMiddleman, + PathFragment dynamicRuntimeSolibDir, + CppCompilationContext cppCompilationContext, + boolean supportsParamFiles, + boolean supportsHeaderParsing) { + this.cppConfiguration = cppConfiguration; + this.crosstool = Preconditions.checkNotNull(crosstool); + this.crosstoolMiddleman = Preconditions.checkNotNull(crosstoolMiddleman); + this.compile = Preconditions.checkNotNull(compile); + this.strip = Preconditions.checkNotNull(strip); + this.objCopy = Preconditions.checkNotNull(objCopy); + this.link = Preconditions.checkNotNull(link); + this.dwp = Preconditions.checkNotNull(dwp); + this.libcLink = Preconditions.checkNotNull(libcLink); + this.staticRuntimeLinkInputs = Preconditions.checkNotNull(staticRuntimeLinkInputs); + this.staticRuntimeLinkMiddleman = staticRuntimeLinkMiddleman; + this.dynamicRuntimeLinkInputs = Preconditions.checkNotNull(dynamicRuntimeLinkInputs); + this.dynamicRuntimeLinkMiddleman = dynamicRuntimeLinkMiddleman; + this.dynamicRuntimeSolibDir = Preconditions.checkNotNull(dynamicRuntimeSolibDir); + this.cppCompilationContext = Preconditions.checkNotNull(cppCompilationContext); + this.supportsParamFiles = supportsParamFiles; + this.supportsHeaderParsing = supportsHeaderParsing; + } + + /** + * Returns all the files in Crosstool. Is not a middleman. + */ + public NestedSet<Artifact> getCrosstool() { + return crosstool; + } + + /** + * Returns a middleman for all the files in Crosstool. + */ + public NestedSet<Artifact> getCrosstoolMiddleman() { + return crosstoolMiddleman; + } + + /** + * Returns the files necessary for compilation. + */ + public NestedSet<Artifact> getCompile() { + // If include scanning is disabled, we need the entire crosstool filegroup, including header + // files. If it is enabled, we use the filegroup without header files - they are found by + // include scanning. For go, we also don't need the header files. + return cppConfiguration != null && cppConfiguration.shouldScanIncludes() ? compile : crosstool; + } + + /** + * Returns the files necessary for a 'strip' invocation. + */ + public NestedSet<Artifact> getStrip() { + return strip; + } + + /** + * Returns the files necessary for an 'objcopy' invocation. + */ + public NestedSet<Artifact> getObjcopy() { + return objCopy; + } + + /** + * Returns the files necessary for linking, including the files needed for libc. + */ + public NestedSet<Artifact> getLink() { + return link; + } + + public NestedSet<Artifact> getDwp() { + return dwp; + } + + public NestedSet<Artifact> getLibcLink() { + return libcLink; + } + + /** + * Returns the static runtime libraries. + */ + public NestedSet<Artifact> getStaticRuntimeLinkInputs() { + return staticRuntimeLinkInputs; + } + + /** + * Returns an aggregating middleman that represents the static runtime libraries. + */ + @Nullable public Artifact getStaticRuntimeLinkMiddleman() { + return staticRuntimeLinkMiddleman; + } + + /** + * Returns the dynamic runtime libraries. + */ + public NestedSet<Artifact> getDynamicRuntimeLinkInputs() { + return dynamicRuntimeLinkInputs; + } + + /** + * Returns an aggregating middleman that represents the dynamic runtime libraries. + */ + @Nullable public Artifact getDynamicRuntimeLinkMiddleman() { + return dynamicRuntimeLinkMiddleman; + } + + /** + * Returns the name of the directory where the solib symlinks for the dynamic runtime libraries + * live. The directory itself will be under the root of the host configuration in the 'bin' + * directory. + */ + public PathFragment getDynamicRuntimeSolibDir() { + return dynamicRuntimeSolibDir; + } + + /** + * Returns the C++ compilation context for the toolchain. + */ + public CppCompilationContext getCppCompilationContext() { + return cppCompilationContext; + } + + /** + * Whether the toolchains supports parameter files. + */ + public boolean supportsParamFiles() { + return supportsParamFiles; + } + + /** + * Whether the toolchains supports header parsing. + */ + public boolean supportsHeaderParsing() { + return supportsHeaderParsing; + } + + /** + * Returns the configured features of the toolchain. + */ + public CcToolchainFeatures getFeatures() { + return cppConfiguration.getFeatures(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java new file mode 100644 index 0000000000..6c68f00bab --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java @@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.LICENSE; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.Attribute.LateBoundLabel; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.syntax.Label; + +/** + * Rule definition for compiler definition. + */ +@BlazeRule(name = "cc_toolchain", + ancestors = { BaseRuleClasses.BaseRule.class }, + factoryClass = CcToolchain.class) +public final class CcToolchainRule implements RuleDefinition { + private static final LateBoundLabel<BuildConfiguration> LIBC_LINK = + new LateBoundLabel<BuildConfiguration>() { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.getFragment(CppConfiguration.class).getLibcLabel(); + } + }; + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .setUndocumented() + .add(attr("output_licenses", LICENSE)) + .add(attr("cpu", STRING).mandatory()) + .add(attr("all_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("compiler_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("strip_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("objcopy_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("linker_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("dwp_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("static_runtime_libs", LABEL_LIST).legacyAllowAnyFileType().mandatory()) + .add(attr("dynamic_runtime_libs", LABEL_LIST).legacyAllowAnyFileType().mandatory()) + .add(attr("module_map", LABEL).legacyAllowAnyFileType().cfg(HOST)) + .add(attr("supports_param_files", BOOLEAN).value(true)) + .add(attr("supports_header_parsing", BOOLEAN).value(false)) + // TODO(bazel-team): Should be using the TARGET configuration. + .add(attr(":libc_link", LABEL).cfg(HOST).value(LIBC_LINK)) + .build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppBuildInfo.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppBuildInfo.java new file mode 100644 index 0000000000..78a5f89700 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppBuildInfo.java @@ -0,0 +1,89 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoCollection; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * C++ build info creation - generates header files that contain the corresponding build-info data. + */ +public final class CppBuildInfo implements BuildInfoFactory { + public static final BuildInfoKey KEY = new BuildInfoKey("C++"); + + private static final PathFragment BUILD_INFO_NONVOLATILE_HEADER_NAME = + new PathFragment("build-info-nonvolatile.h"); + private static final PathFragment BUILD_INFO_VOLATILE_HEADER_NAME = + new PathFragment("build-info-volatile.h"); + // TODO(bazel-team): (2011) Get rid of the redacted build info. We should try to make + // the linkstamping process handle the case where those values are undefined. + private static final PathFragment BUILD_INFO_REDACTED_HEADER_NAME = + new PathFragment("build-info-redacted.h"); + + @Override + public BuildInfoCollection create(BuildInfoContext buildInfoContext, BuildConfiguration config, + Artifact buildInfo, Artifact buildChangelist) { + List<Action> actions = new ArrayList<>(); + WriteBuildInfoHeaderAction redactedInfo = getHeader(buildInfoContext, config, + BUILD_INFO_REDACTED_HEADER_NAME, + Artifact.NO_ARTIFACTS, true, true); + WriteBuildInfoHeaderAction nonvolatileInfo = getHeader(buildInfoContext, config, + BUILD_INFO_NONVOLATILE_HEADER_NAME, + ImmutableList.of(buildInfo), + false, true); + WriteBuildInfoHeaderAction volatileInfo = getHeader(buildInfoContext, config, + BUILD_INFO_VOLATILE_HEADER_NAME, + ImmutableList.of(buildChangelist), + true, false); + actions.add(redactedInfo); + actions.add(nonvolatileInfo); + actions.add(volatileInfo); + return new BuildInfoCollection(actions, + ImmutableList.of(nonvolatileInfo.getPrimaryOutput(), volatileInfo.getPrimaryOutput()), + ImmutableList.of(redactedInfo.getPrimaryOutput())); + } + + private WriteBuildInfoHeaderAction getHeader(BuildInfoContext buildInfoContext, + BuildConfiguration config, PathFragment headerName, + Collection<Artifact> inputs, + boolean writeVolatileInfo, boolean writeNonVolatileInfo) { + Root outputPath = config.getIncludeDirectory(); + final Artifact header = + buildInfoContext.getBuildInfoArtifact(headerName, outputPath, + writeVolatileInfo && !inputs.isEmpty() + ? BuildInfoType.NO_REBUILD : BuildInfoType.FORCE_REBUILD_IF_CHANGED); + return new WriteBuildInfoHeaderAction( + inputs, header, writeVolatileInfo, writeNonVolatileInfo); + } + + @Override + public BuildInfoKey getKey() { + return KEY; + } + + @Override + public boolean isEnabled(BuildConfiguration config) { + return config.hasFragment(CppConfiguration.class); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompilationContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompilationContext.java new file mode 100644 index 0000000000..cf39ef57c5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompilationContext.java @@ -0,0 +1,918 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.MiddlemanFactory; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Immutable store of information needed for C++ compilation that is aggregated + * across dependencies. + */ +@Immutable +public final class CppCompilationContext implements TransitiveInfoProvider { + /** An empty compilation context. */ + public static final CppCompilationContext EMPTY = new Builder(null).build(); + + private final CommandLineContext commandLineContext; + private final ImmutableList<DepsContext> depsContexts; + private final CppModuleMap cppModuleMap; + private final Artifact headerModule; + private final Artifact picHeaderModule; + private final ImmutableSet<Artifact> compilationPrerequisites; + + private CppCompilationContext(CommandLineContext commandLineContext, + List<DepsContext> depsContexts, CppModuleMap cppModuleMap, Artifact headerModule, + Artifact picHeaderModule) { + Preconditions.checkNotNull(commandLineContext); + Preconditions.checkArgument(!depsContexts.isEmpty()); + this.commandLineContext = commandLineContext; + this.depsContexts = ImmutableList.copyOf(depsContexts); + this.cppModuleMap = cppModuleMap; + this.headerModule = headerModule; + this.picHeaderModule = picHeaderModule; + + if (depsContexts.size() == 1) { + // Only LIPO targets have more than one DepsContexts. This codepath avoids creating + // an ImmutableSet.Builder for the vast majority of the cases. + compilationPrerequisites = (depsContexts.get(0).compilationPrerequisiteStampFile != null) + ? ImmutableSet.<Artifact>of(depsContexts.get(0).compilationPrerequisiteStampFile) + : ImmutableSet.<Artifact>of(); + } else { + ImmutableSet.Builder<Artifact> prerequisites = ImmutableSet.builder(); + for (DepsContext depsContext : depsContexts) { + if (depsContext.compilationPrerequisiteStampFile != null) { + prerequisites.add(depsContext.compilationPrerequisiteStampFile); + } + } + compilationPrerequisites = prerequisites.build(); + } + } + + /** + * Returns the compilation prerequisites consolidated into middlemen + * prerequisites, or an empty set if there are no prerequisites. + * + * <p>For correct dependency tracking, and to reduce the overhead to establish + * dependencies on generated headers, we express the dependency on compilation + * prerequisites as a transitive dependency via a middleman. After they have + * been accumulated (using + * {@link Builder#addCompilationPrerequisites(Iterable)}, + * {@link Builder#mergeDependentContext(CppCompilationContext)}, and + * {@link Builder#mergeDependentContexts(Iterable)}, they are consolidated + * into a single middleman Artifact when {@link Builder#build()} is called. + * + * <p>The returned set can be empty if there are no prerequisites. Usually it + * contains a single middleman, but if LIPO is used there can be two. + */ + public ImmutableSet<Artifact> getCompilationPrerequisites() { + return compilationPrerequisites; + } + + /** + * Returns the immutable list of include directories to be added with "-I" + * (possibly empty but never null). This includes the include dirs from the + * transitive deps closure of the target. This list does not contain + * duplicates. All fragments are either absolute or relative to the exec root + * (see {@link BuildConfiguration#getExecRoot}). + */ + public ImmutableList<PathFragment> getIncludeDirs() { + return commandLineContext.includeDirs; + } + + /** + * Returns the immutable list of include directories to be added with + * "-iquote" (possibly empty but never null). This includes the include dirs + * from the transitive deps closure of the target. This list does not contain + * duplicates. All fragments are either absolute or relative to the exec root + * (see {@link BuildConfiguration#getExecRoot}). + */ + public ImmutableList<PathFragment> getQuoteIncludeDirs() { + return commandLineContext.quoteIncludeDirs; + } + + /** + * Returns the immutable list of include directories to be added with + * "-isystem" (possibly empty but never null). This includes the include dirs + * from the transitive deps closure of the target. This list does not contain + * duplicates. All fragments are either absolute or relative to the exec root + * (see {@link BuildConfiguration#getExecRoot}). + */ + public ImmutableList<PathFragment> getSystemIncludeDirs() { + return commandLineContext.systemIncludeDirs; + } + + /** + * Returns the immutable set of declared include directories, relative to a + * "-I" or "-iquote" directory" (possibly empty but never null). The returned + * collection may contain duplicate elements. + * + * <p>Note: The iteration order of this list is preserved as ide_build_info + * writes these directories and sources out and the ordering will help when + * used by consumers. + */ + public NestedSet<PathFragment> getDeclaredIncludeDirs() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).declaredIncludeDirs; + } + + NestedSetBuilder<PathFragment> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.declaredIncludeDirs); + } + + return builder.build(); + } + + /** + * Returns the immutable set of include directories, relative to a "-I" or + * "-iquote" directory", from which inclusion will produce a warning (possibly + * empty but never null). The returned collection may contain duplicate + * elements. + * + * <p>Note: The iteration order of this list is preserved as ide_build_info + * writes these directories and sources out and the ordering will help when + * used by consumers. + */ + public NestedSet<PathFragment> getDeclaredIncludeWarnDirs() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).declaredIncludeWarnDirs; + } + + NestedSetBuilder<PathFragment> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.declaredIncludeWarnDirs); + } + + return builder.build(); + } + + /** + * Returns the immutable set of headers that have been declared in the + * {@code src} or {@code headers attribute} (possibly empty but never null). + * The returned collection may contain duplicate elements. + * + * <p>Note: The iteration order of this list is preserved as ide_build_info + * writes these directories and sources out and the ordering will help when + * used by consumers. + */ + public NestedSet<Artifact> getDeclaredIncludeSrcs() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).declaredIncludeSrcs; + } + + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.declaredIncludeSrcs); + } + + return builder.build(); + } + + /** + * Returns the immutable pairs of (header file, pregrepped header file). + */ + public NestedSet<Pair<Artifact, Artifact>> getPregreppedHeaders() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).pregreppedHdrs; + } + + NestedSetBuilder<Pair<Artifact, Artifact>> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.pregreppedHdrs); + } + + return builder.build(); + } + + /** + * Returns the immutable set of additional transitive inputs needed for + * compilation, like C++ module map artifacts. + */ + public NestedSet<Artifact> getAdditionalInputs() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).auxiliaryInputs; + } + + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.auxiliaryInputs); + } + + return builder.build(); + } + + /** + * Returns optional inputs that are needed by any C++ compilations that use header modules. + * + * <p>For every target that the current target depends on transitively and that is built as header + * module, contains: + * <ul> + * <li>the pic/non-pic header module (pcm file)</li> + * <li>the transitive list of module maps.</li> + * </ul> + */ + private NestedSet<Artifact> getTransitiveAuxiliaryInputs() { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.transitiveAuxiliaryInputs); + } + return builder.build(); + } + + /** + * @return all modules maps in the transitive closure. + */ + private NestedSet<Artifact> getTransitiveModuleMaps() { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.transitiveModuleMaps); + } + return builder.build(); + } + + /** + * @return all headers whose transitive closure of includes needs to be + * available when compiling anything in the current target. + */ + protected NestedSet<Artifact> getTransitiveHeaderModuleSrcs() { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.transitiveHeaderModuleSrcs); + } + return builder.build(); + } + + /** + * @return all declared headers of the current module if the current target + * is compiled as a module. + */ + protected NestedSet<Artifact> getHeaderModuleSrcs() { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.headerModuleSrcs); + } + return builder.build(); + } + + /** + * Returns the set of defines needed to compile this target (possibly empty + * but never null). This includes definitions from the transitive deps closure + * for the target. The order of the returned collection is deterministic. + */ + public ImmutableList<String> getDefines() { + return commandLineContext.defines; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof CppCompilationContext)) { + return false; + } + CppCompilationContext other = (CppCompilationContext) obj; + return Objects.equals(headerModule, other.headerModule) + && Objects.equals(picHeaderModule, other.picHeaderModule) + && commandLineContext.equals(other.commandLineContext) + && depsContexts.equals(other.depsContexts); + } + + @Override + public int hashCode() { + return Objects.hash(headerModule, picHeaderModule, commandLineContext, depsContexts); + } + + /** + * Returns a context that is based on a given context but returns empty sets + * for {@link #getDeclaredIncludeDirs()} and {@link #getDeclaredIncludeWarnDirs()}. + */ + public static CppCompilationContext disallowUndeclaredHeaders(CppCompilationContext context) { + ImmutableList.Builder<DepsContext> builder = ImmutableList.builder(); + for (DepsContext depsContext : context.depsContexts) { + builder.add(new DepsContext( + depsContext.compilationPrerequisiteStampFile, + NestedSetBuilder.<PathFragment>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<PathFragment>emptySet(Order.STABLE_ORDER), + depsContext.declaredIncludeSrcs, + depsContext.pregreppedHdrs, + depsContext.auxiliaryInputs, + depsContext.headerModuleSrcs, + depsContext.transitiveAuxiliaryInputs, + depsContext.transitiveHeaderModuleSrcs, + depsContext.transitiveModuleMaps)); + } + return new CppCompilationContext(context.commandLineContext, builder.build(), + context.cppModuleMap, context.headerModule, context.picHeaderModule); + } + + /** + * Returns the context for a LIPO compile action. This uses the include dirs + * and defines of the library, but the declared inclusion dirs/srcs from both + * the library and the owner binary. + + * TODO(bazel-team): this might make every LIPO target have an unnecessary large set of + * inclusion dirs/srcs. The correct behavior would be to merge only the contexts + * of actual referred targets (as listed in .imports file). + * + * <p>Undeclared inclusion checking ({@link #getDeclaredIncludeDirs()}, + * {@link #getDeclaredIncludeWarnDirs()}, and + * {@link #getDeclaredIncludeSrcs()}) needs to use the union of the contexts + * of the involved source files. + * + * <p>For include and define command line flags ({@link #getIncludeDirs()} + * {@link #getQuoteIncludeDirs()}, {@link #getSystemIncludeDirs()}, and + * {@link #getDefines()}) LIPO compilations use the same values as non-LIPO + * compilation. + * + * <p>Include scanning is not handled by this method. See + * {@code IncludeScannable#getAuxiliaryScannables()} instead. + * + * @param ownerContext the compilation context of the owner binary + * @param libContext the compilation context of the library + */ + public static CppCompilationContext mergeForLipo(CppCompilationContext ownerContext, + CppCompilationContext libContext) { + return new CppCompilationContext(libContext.commandLineContext, + ImmutableList.copyOf(Iterables.concat(ownerContext.depsContexts, libContext.depsContexts)), + libContext.cppModuleMap, libContext.headerModule, libContext.picHeaderModule); + } + + /** + * @return the C++ module map of the owner. + */ + public CppModuleMap getCppModuleMap() { + return cppModuleMap; + } + + /** + * @return the non-pic C++ header module of the owner. + */ + private Artifact getHeaderModule() { + return headerModule; + } + + /** + * @return the pic C++ header module of the owner. + */ + private Artifact getPicHeaderModule() { + return picHeaderModule; + } + + /** + * The parts of the compilation context that influence the command line of + * compilation actions. + */ + @Immutable + private static class CommandLineContext { + private final ImmutableList<PathFragment> includeDirs; + private final ImmutableList<PathFragment> quoteIncludeDirs; + private final ImmutableList<PathFragment> systemIncludeDirs; + private final ImmutableList<String> defines; + + CommandLineContext(ImmutableList<PathFragment> includeDirs, + ImmutableList<PathFragment> quoteIncludeDirs, + ImmutableList<PathFragment> systemIncludeDirs, + ImmutableList<String> defines) { + this.includeDirs = includeDirs; + this.quoteIncludeDirs = quoteIncludeDirs; + this.systemIncludeDirs = systemIncludeDirs; + this.defines = defines; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof CommandLineContext)) { + return false; + } + CommandLineContext other = (CommandLineContext) obj; + return Objects.equals(includeDirs, other.includeDirs) + && Objects.equals(quoteIncludeDirs, other.quoteIncludeDirs) + && Objects.equals(systemIncludeDirs, other.systemIncludeDirs) + && Objects.equals(defines, other.defines); + } + + @Override + public int hashCode() { + return Objects.hash(includeDirs, quoteIncludeDirs, systemIncludeDirs, defines); + } + } + + /** + * The parts of the compilation context that defined the dependencies of + * actions of scheduling and inclusion validity checking. + */ + @Immutable + private static class DepsContext { + private final Artifact compilationPrerequisiteStampFile; + private final NestedSet<PathFragment> declaredIncludeDirs; + private final NestedSet<PathFragment> declaredIncludeWarnDirs; + private final NestedSet<Artifact> declaredIncludeSrcs; + private final NestedSet<Pair<Artifact, Artifact>> pregreppedHdrs; + + /** + * Optional inputs that are used by some forms of compilation, containing: + * <ul> + * <li>module map of the current target</li> + * <li>module maps of all direct dependencies that are not compiled as header modules</li> + * <li>all transitiveAuxiliaryInputs.</li> + * </ul> + */ + private final NestedSet<Artifact> auxiliaryInputs; + + /** + * All declared headers of the current module, if compiled as a header module. + */ + private final NestedSet<Artifact> headerModuleSrcs; + + private final NestedSet<Artifact> transitiveAuxiliaryInputs; + + /** + * Headers whose transitive closure of includes needs to be available when compiling the current + * target. For every target that the current target depends on transitively and that is built as + * header module, contains all headers that are part of its header module. + */ + private final NestedSet<Artifact> transitiveHeaderModuleSrcs; + + /** + * The module maps from all targets the current target depends on transitively. + */ + private final NestedSet<Artifact> transitiveModuleMaps; + + DepsContext(Artifact compilationPrerequisiteStampFile, + NestedSet<PathFragment> declaredIncludeDirs, + NestedSet<PathFragment> declaredIncludeWarnDirs, + NestedSet<Artifact> declaredIncludeSrcs, + NestedSet<Pair<Artifact, Artifact>> pregreppedHdrs, + NestedSet<Artifact> auxiliaryInputs, + NestedSet<Artifact> headerModuleSrcs, + NestedSet<Artifact> transitiveAuxiliaryInputs, + NestedSet<Artifact> transitiveHeaderModuleSrcs, + NestedSet<Artifact> transitiveModuleMaps) { + this.compilationPrerequisiteStampFile = compilationPrerequisiteStampFile; + this.declaredIncludeDirs = declaredIncludeDirs; + this.declaredIncludeWarnDirs = declaredIncludeWarnDirs; + this.declaredIncludeSrcs = declaredIncludeSrcs; + this.pregreppedHdrs = pregreppedHdrs; + this.auxiliaryInputs = auxiliaryInputs; + this.headerModuleSrcs = headerModuleSrcs; + this.transitiveAuxiliaryInputs = transitiveAuxiliaryInputs; + this.transitiveHeaderModuleSrcs = transitiveHeaderModuleSrcs; + this.transitiveModuleMaps = transitiveModuleMaps; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof DepsContext)) { + return false; + } + DepsContext other = (DepsContext) obj; + return Objects.equals( + compilationPrerequisiteStampFile, other.compilationPrerequisiteStampFile) + && Objects.equals(declaredIncludeDirs, other.declaredIncludeDirs) + && Objects.equals(declaredIncludeWarnDirs, other.declaredIncludeWarnDirs) + && Objects.equals(declaredIncludeSrcs, other.declaredIncludeSrcs) + && Objects.equals(auxiliaryInputs, other.auxiliaryInputs) + && Objects.equals(headerModuleSrcs, other.headerModuleSrcs) + // Due to the NestedSet equals being ==, and the code flow only setting them if at least + // auxiliaryInputs is set, these checks cannot be executed. We leave them in so the equals + // is still correct if that connection ever changes.R + && Objects.equals(transitiveAuxiliaryInputs, other.transitiveAuxiliaryInputs) + && Objects.equals(transitiveHeaderModuleSrcs, other.transitiveHeaderModuleSrcs) + && Objects.equals(transitiveModuleMaps, other.transitiveModuleMaps) + ; + } + + @Override + public int hashCode() { + return Objects.hash(compilationPrerequisiteStampFile, + declaredIncludeDirs, + declaredIncludeWarnDirs, + declaredIncludeSrcs, + auxiliaryInputs, + headerModuleSrcs, + transitiveAuxiliaryInputs, + transitiveHeaderModuleSrcs, + transitiveModuleMaps); + } + } + + /** + * Builder class for {@link CppCompilationContext}. + */ + public static class Builder { + private String purpose = "cpp_compilation_prerequisites"; + private final Set<Artifact> compilationPrerequisites = new LinkedHashSet<>(); + private final Set<PathFragment> includeDirs = new LinkedHashSet<>(); + private final Set<PathFragment> quoteIncludeDirs = new LinkedHashSet<>(); + private final Set<PathFragment> systemIncludeDirs = new LinkedHashSet<>(); + private final NestedSetBuilder<PathFragment> declaredIncludeDirs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<PathFragment> declaredIncludeWarnDirs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<Artifact> declaredIncludeSrcs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<Pair<Artifact, Artifact>> pregreppedHdrs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<Artifact> auxiliaryInputs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<Artifact> headerModuleSrcs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<Artifact> transitiveAuxiliaryInputs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<Artifact> transitiveHeaderModuleSrcs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<Artifact> transitiveModuleMaps = + NestedSetBuilder.stableOrder(); + private final Set<String> defines = new LinkedHashSet<>(); + private CppModuleMap cppModuleMap; + private Artifact headerModule; + private Artifact picHeaderModule; + + /** The rule that owns the context */ + private final RuleContext ruleContext; + + /** + * Creates a new builder for a {@link CppCompilationContext} instance. + */ + public Builder(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + /** + * Overrides the purpose of this context. This is useful if a Target + * needs more than one CppCompilationContext. (The purpose is used to + * construct the name of the prerequisites middleman for the context, and + * all artifacts for a given Target must have distinct names.) + * + * @param purpose must be a string which is suitable for use as a filename. + * A single rule may have many middlemen with distinct purposes. + * + * @see MiddlemanFactory#createErrorPropagatingMiddleman + */ + public Builder setPurpose(String purpose) { + this.purpose = purpose; + return this; + } + + public String getPurpose() { + return purpose; + } + + /** + * Merges the context of a dependency into this one by adding the contents + * of all of its attributes. + */ + public Builder mergeDependentContext(CppCompilationContext otherContext) { + Preconditions.checkNotNull(otherContext); + compilationPrerequisites.addAll(otherContext.getCompilationPrerequisites()); + includeDirs.addAll(otherContext.getIncludeDirs()); + quoteIncludeDirs.addAll(otherContext.getQuoteIncludeDirs()); + systemIncludeDirs.addAll(otherContext.getSystemIncludeDirs()); + declaredIncludeDirs.addTransitive(otherContext.getDeclaredIncludeDirs()); + declaredIncludeWarnDirs.addTransitive(otherContext.getDeclaredIncludeWarnDirs()); + declaredIncludeSrcs.addTransitive(otherContext.getDeclaredIncludeSrcs()); + pregreppedHdrs.addTransitive(otherContext.getPregreppedHeaders()); + + // Forward transitive information. + transitiveAuxiliaryInputs.addTransitive(otherContext.getTransitiveAuxiliaryInputs()); + transitiveModuleMaps.addTransitive(otherContext.getTransitiveModuleMaps()); + transitiveHeaderModuleSrcs.addTransitive(otherContext.getTransitiveHeaderModuleSrcs()); + + // All module maps of direct dependencies are inputs to the current compile independently of + // the build type. + if (otherContext.getCppModuleMap() != null) { + auxiliaryInputs.add(otherContext.getCppModuleMap().getArtifact()); + } + if (otherContext.getHeaderModule() != null || otherContext.getPicHeaderModule() != null) { + // If we depend directly on a target that has a compiled header module, all targets + // transitively depending on us will need that header module, and all transitive module + // maps. + if (otherContext.getHeaderModule() != null) { + transitiveAuxiliaryInputs.add(otherContext.getHeaderModule()); + } + if (otherContext.getPicHeaderModule() != null) { + transitiveAuxiliaryInputs.add(otherContext.getPicHeaderModule()); + } + transitiveAuxiliaryInputs.addAll(otherContext.getTransitiveModuleMaps()); + + // All targets transitively depending on us will need to have the full transitive #include + // closure of the headers in that module available. + transitiveHeaderModuleSrcs.addAll(otherContext.getHeaderModuleSrcs()); + } + // All compile actions in the current target will need the transitive inputs. + auxiliaryInputs.addAll(transitiveAuxiliaryInputs.build().toCollection()); + + defines.addAll(otherContext.getDefines()); + return this; + } + + /** + * Merges the context of some targets into this one by adding the contents + * of all of their attributes. Targets that do not implement + * {@link CppCompilationContext} are ignored. + */ + public Builder mergeDependentContexts(Iterable<CppCompilationContext> targets) { + for (CppCompilationContext target : targets) { + mergeDependentContext(target); + } + return this; + } + + /** + * Adds multiple compilation prerequisites. + */ + public Builder addCompilationPrerequisites(Iterable<Artifact> prerequisites) { + // LIPO collector must not add compilation prerequisites in order to avoid + // the creation of a middleman action. + Iterables.addAll(compilationPrerequisites, prerequisites); + return this; + } + + /** + * Add a single include directory to be added with "-I". It can be either + * relative to the exec root (see {@link BuildConfiguration#getExecRoot}) or + * absolute. Before it is stored, the include directory is normalized. + */ + public Builder addIncludeDir(PathFragment includeDir) { + includeDirs.add(includeDir.normalize()); + return this; + } + + /** + * Add multiple include directories to be added with "-I". These can be + * either relative to the exec root (see {@link + * BuildConfiguration#getExecRoot}) or absolute. The entries are normalized + * before they are stored. + */ + public Builder addIncludeDirs(Iterable<PathFragment> includeDirs) { + for (PathFragment includeDir : includeDirs) { + addIncludeDir(includeDir); + } + return this; + } + + /** + * Add a single include directory to be added with "-iquote". It can be + * either relative to the exec root (see {@link + * BuildConfiguration#getExecRoot}) or absolute. Before it is stored, the + * include directory is normalized. + */ + public Builder addQuoteIncludeDir(PathFragment quoteIncludeDir) { + quoteIncludeDirs.add(quoteIncludeDir.normalize()); + return this; + } + + /** + * Add a single include directory to be added with "-isystem". It can be + * either relative to the exec root (see {@link + * BuildConfiguration#getExecRoot}) or absolute. Before it is stored, the + * include directory is normalized. + */ + public Builder addSystemIncludeDir(PathFragment systemIncludeDir) { + systemIncludeDirs.add(systemIncludeDir.normalize()); + return this; + } + + /** + * Add a single declared include dir, relative to a "-I" or "-iquote" + * directory". + */ + public Builder addDeclaredIncludeDir(PathFragment dir) { + declaredIncludeDirs.add(dir); + return this; + } + + /** + * Add a single declared include directory, relative to a "-I" or "-iquote" + * directory", from which inclusion will produce a warning. + */ + public Builder addDeclaredIncludeWarnDir(PathFragment dir) { + declaredIncludeWarnDirs.add(dir); + return this; + } + + /** + * Adds a header that has been declared in the {@code src} or {@code headers attribute}. The + * header will also be added to the compilation prerequisites. + */ + public Builder addDeclaredIncludeSrc(Artifact header) { + declaredIncludeSrcs.add(header); + compilationPrerequisites.add(header); + headerModuleSrcs.add(header); + return this; + } + + /** + * Adds multiple headers that have been declared in the {@code src} or {@code headers + * attribute}. The headers will also be added to the compilation prerequisites. + */ + public Builder addDeclaredIncludeSrcs(Iterable<Artifact> declaredIncludeSrcs) { + this.declaredIncludeSrcs.addAll(declaredIncludeSrcs); + this.headerModuleSrcs.addAll(declaredIncludeSrcs); + return addCompilationPrerequisites(declaredIncludeSrcs); + } + + /** + * Add a map of generated source or header Artifact to an output Artifact after grepping + * the file for include statements. + */ + public Builder addPregreppedHeaderMap(Map<Artifact, Artifact> pregrepped) { + addCompilationPrerequisites(pregrepped.values()); + for (Map.Entry<Artifact, Artifact> entry : pregrepped.entrySet()) { + this.pregreppedHdrs.add(Pair.of(entry.getKey(), entry.getValue())); + } + return this; + } + + /** + * Adds a single define. + */ + public Builder addDefine(String define) { + defines.add(define); + return this; + } + + /** + * Adds multiple defines. + */ + public Builder addDefines(Iterable<String> defines) { + Iterables.addAll(this.defines, defines); + return this; + } + + /** + * Sets the C++ module map. + */ + public Builder setCppModuleMap(CppModuleMap cppModuleMap) { + this.cppModuleMap = cppModuleMap; + return this; + } + + /** + * Sets the C++ header module in non-pic mode. + */ + public Builder setHeaderModule(Artifact headerModule) { + this.headerModule = headerModule; + return this; + } + + /** + * Sets the C++ header module in pic mode. + */ + public Builder setPicHeaderModule(Artifact picHeaderModule) { + this.picHeaderModule = picHeaderModule; + return this; + } + + /** + * Builds the {@link CppCompilationContext}. + */ + public CppCompilationContext build() { + return build( + ruleContext == null ? null : ruleContext.getActionOwner(), + ruleContext == null ? null : ruleContext.getAnalysisEnvironment().getMiddlemanFactory()); + } + + @VisibleForTesting // productionVisibility = Visibility.PRIVATE + public CppCompilationContext build(ActionOwner owner, MiddlemanFactory middlemanFactory) { + if (cppModuleMap != null) { + // .cppmap files should also be mandatory inputs for compile actions + auxiliaryInputs.add(cppModuleMap.getArtifact()); + transitiveModuleMaps.add(cppModuleMap.getArtifact()); + } + + // We don't create middlemen in LIPO collector subtree, because some target CT + // will do that instead. + Artifact prerequisiteStampFile = (ruleContext != null + && ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector()) + ? getMiddlemanArtifact(middlemanFactory) + : createMiddleman(owner, middlemanFactory); + + return new CppCompilationContext( + new CommandLineContext(ImmutableList.copyOf(includeDirs), + ImmutableList.copyOf(quoteIncludeDirs), ImmutableList.copyOf(systemIncludeDirs), + ImmutableList.copyOf(defines)), + ImmutableList.of(new DepsContext(prerequisiteStampFile, + declaredIncludeDirs.build(), + declaredIncludeWarnDirs.build(), + declaredIncludeSrcs.build(), + pregreppedHdrs.build(), + auxiliaryInputs.build(), + headerModuleSrcs.build(), + transitiveAuxiliaryInputs.build(), + transitiveHeaderModuleSrcs.build(), + transitiveModuleMaps.build())), + cppModuleMap, + headerModule, + picHeaderModule); + } + + /** + * Creates a middleman for the compilation prerequisites. + * + * @return the middleman or null if there are no prerequisites + */ + private Artifact createMiddleman(ActionOwner owner, + MiddlemanFactory middlemanFactory) { + if (compilationPrerequisites.isEmpty()) { + return null; + } + + // Compilation prerequisites gathered in the compilationPrerequisites + // must be generated prior to executing C++ compilation step that depends + // on them (since these prerequisites include all potential header files, etc + // that could be referenced during compilation). So there is a definite need + // to ensure scheduling edge dependency. However, those prerequisites should + // have no effect on the decision whether C++ compilation should happen in + // the first place - only CppCompileAction outputs (*.o and *.d files) and + // all files referenced by the *.d file should be used to make that decision. + // If this action was never executed, then *.d file would be missing, forcing + // compilation to occur. If *.d file is present and has not changed then the + // only reason that would force us to re-compile would be change in one of + // the files referenced by the *.d file, since no other files participated + // in the compilation. We also need to propagate errors through this + // dependency link. So we use an error propagating middleman. + // Such middleman will be ignored by the dependency checker yet will still + // represent an edge in the action dependency graph - forcing proper execution + // order and error propagation. + return middlemanFactory.createErrorPropagatingMiddleman( + owner, ruleContext.getLabel().toString(), purpose, + ImmutableList.copyOf(compilationPrerequisites), + ruleContext.getConfiguration().getMiddlemanDirectory()); + } + + /** + * Returns the same set of artifacts as createMiddleman() would, but without + * actually creating middlemen. + */ + private Artifact getMiddlemanArtifact(MiddlemanFactory middlemanFactory) { + if (compilationPrerequisites.isEmpty()) { + return null; + } + + return middlemanFactory.getErrorPropagatingMiddlemanArtifact(ruleContext.getLabel() + .toString(), purpose, ruleContext.getConfiguration().getMiddlemanDirectory()); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java new file mode 100644 index 0000000000..e90f9f74ba --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java @@ -0,0 +1,1356 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Artifact.MiddlemanExpander; +import com.google.devtools.build.lib.actions.ArtifactResolver; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.extra.CppCompileInfo; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.PerLabelOptions; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.Tool; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.DependencySet; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import javax.annotation.Nullable; + +/** + * Action that represents some kind of C++ compilation step. + */ +@ThreadCompatible +public class CppCompileAction extends AbstractAction implements IncludeScannable { + /** + * Represents logic that determines which artifacts, if any, should be added to the actual inputs + * for each included file (in addition to the included file itself) + */ + public interface IncludeResolver { + /** + * Returns the set of files to be added for an included file (as returned in the .d file) + */ + Iterable<Artifact> getInputsForIncludedFile( + Artifact includedFile, ArtifactResolver artifactResolver); + } + + public static final IncludeResolver VOID_INCLUDE_RESOLVER = new IncludeResolver() { + @Override + public Iterable<Artifact> getInputsForIncludedFile(Artifact includedFile, + ArtifactResolver artifactResolver) { + return ImmutableList.of(); + } + }; + + private static final int VALIDATION_DEBUG = 0; // 0==none, 1==warns/errors, 2==all + private static final boolean VALIDATION_DEBUG_WARN = VALIDATION_DEBUG >= 1; + + /** + * A string constant for the c compilation action. + */ + public static final String C_COMPILE = "c-compile"; + + /** + * A string constant for the c++ compilation action. + */ + public static final String CPP_COMPILE = "c++-compile"; + + /** + * A string constant for the c++ header parsing. + */ + public static final String CPP_HEADER_PARSING = "c++-header-parsing"; + + /** + * A string constant for the c++ header preprocessing. + */ + public static final String CPP_HEADER_PREPROCESSING = "c++-header-preprocessing"; + + /** + * A string constant for the c++ module compilation action. + * Note: currently we don't support C module compilation. + */ + public static final String CPP_MODULE_COMPILE = "c++-module-compile"; + + /** + * A string constant for the preprocessing assembler action. + */ + public static final String PREPROCESS_ASSEMBLE = "preprocess-assemble"; + + + private final BuildConfiguration configuration; + protected final Artifact outputFile; + private final Label sourceLabel; + private final Artifact dwoFile; + private final Artifact optionalSourceFile; + private final NestedSet<Artifact> mandatoryInputs; + private final CppCompilationContext context; + private final Collection<PathFragment> extraSystemIncludePrefixes; + private final Iterable<IncludeScannable> lipoScannables; + private final CppCompileCommandLine cppCompileCommandLine; + private final boolean enableLayeringCheck; + private final boolean compileHeaderModules; + + @VisibleForTesting + final CppConfiguration cppConfiguration; + private final Class<? extends CppCompileActionContext> actionContext; + private final IncludeResolver includeResolver; + + /** + * Identifier for the actual execution time behavior of the action. + * + * <p>Required because the behavior of this class can be modified by injecting code in the + * constructor or by inheritance, and we want to have different cache keys for those. + */ + private final UUID actionClassId; + + private boolean inputsKnown = false; + + /** + * Set when the action prepares for execution. Used to preserve state between preparation and + * execution. + */ + private Collection<? extends ActionInput> additionalInputs = null; + + /** + * Creates a new action to compile C/C++ source files. + * + * @param owner the owner of the action, usually the configured target that + * emitted it + * @param sourceFile the source file that should be compiled. {@code mandatoryInputs} must + * contain this file + * @param sourceLabel the label of the rule the source file is generated by + * @param mandatoryInputs any additional files that need to be present for the + * compilation to succeed, can be empty but not null, for example, extra sources for FDO. + * @param outputFile the object file that is written as result of the + * compilation, or the fake object for {@link FakeCppCompileAction}s + * @param dotdFile the .d file that is generated as a side-effect of + * compilation + * @param gcnoFile the coverage notes that are written in coverage mode, can + * be null + * @param dwoFile the .dwo output file where debug information is stored for Fission + * builds (null if Fission mode is disabled) + * @param optionalSourceFile an additional optional source file (null if unneeded) + * @param configuration the build configurations + * @param context the compilation context + * @param copts options for the compiler + * @param coptsFilter regular expression to remove options from {@code copts} + * @param compileHeaderModules whether to compile C++ header modules + */ + protected CppCompileAction(ActionOwner owner, + // TODO(bazel-team): Eventually we will remove 'features'; all functionality in 'features' + // will be provided by 'featureConfiguration'. + ImmutableList<String> features, + FeatureConfiguration featureConfiguration, + Artifact sourceFile, + Label sourceLabel, + NestedSet<Artifact> mandatoryInputs, + Artifact outputFile, + DotdFile dotdFile, + @Nullable Artifact gcnoFile, + @Nullable Artifact dwoFile, + Artifact optionalSourceFile, + BuildConfiguration configuration, + CppConfiguration cppConfiguration, + CppCompilationContext context, + Class<? extends CppCompileActionContext> actionContext, + ImmutableList<String> copts, + ImmutableList<String> pluginOpts, + Predicate<String> coptsFilter, + ImmutableList<PathFragment> extraSystemIncludePrefixes, + boolean enableLayeringCheck, + @Nullable String fdoBuildStamp, + IncludeResolver includeResolver, + Iterable<IncludeScannable> lipoScannables, + UUID actionClassId, + boolean compileHeaderModules) { + // getInputs() method is overridden in this class so we pass a dummy empty + // list to the AbstractAction constructor in place of a real input collection. + super(owner, + Artifact.NO_ARTIFACTS, + CollectionUtils.asListWithoutNulls(outputFile, dotdFile.artifact(), + gcnoFile, dwoFile)); + this.configuration = configuration; + this.sourceLabel = sourceLabel; + this.outputFile = Preconditions.checkNotNull(outputFile); + this.dwoFile = dwoFile; + this.optionalSourceFile = optionalSourceFile; + this.context = context; + this.extraSystemIncludePrefixes = extraSystemIncludePrefixes; + this.enableLayeringCheck = enableLayeringCheck; + this.includeResolver = includeResolver; + this.cppConfiguration = cppConfiguration; + if (cppConfiguration != null && !cppConfiguration.shouldScanIncludes()) { + inputsKnown = true; + } + this.cppCompileCommandLine = new CppCompileCommandLine(sourceFile, dotdFile, + context.getCppModuleMap(), copts, coptsFilter, pluginOpts, + (gcnoFile != null), features, featureConfiguration, fdoBuildStamp); + this.actionContext = actionContext; + this.lipoScannables = lipoScannables; + this.actionClassId = actionClassId; + this.compileHeaderModules = compileHeaderModules; + + // We do not need to include the middleman artifact since it is a generated + // artifact and will definitely exist prior to this action execution. + this.mandatoryInputs = mandatoryInputs; + setInputs(createInputs(mandatoryInputs, context.getCompilationPrerequisites(), + optionalSourceFile)); + } + + private static NestedSet<Artifact> createInputs( + NestedSet<Artifact> mandatoryInputs, + Set<Artifact> prerequisites, Artifact optionalSourceFile) { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + if (optionalSourceFile != null) { + builder.add(optionalSourceFile); + } + builder.addAll(prerequisites); + builder.addTransitive(mandatoryInputs); + return builder.build(); + } + + public boolean shouldScanIncludes() { + return cppConfiguration.shouldScanIncludes(); + } + + @Override + public List<PathFragment> getBuiltInIncludeDirectories() { + return cppConfiguration.getBuiltInIncludeDirectories(); + } + + public String getHostSystemName() { + return cppConfiguration.getHostSystemName(); + } + + @Override + public NestedSet<Artifact> getMandatoryInputs() { + return mandatoryInputs; + } + + @Override + public boolean inputsKnown() { + return inputsKnown; + } + + /** + * Returns the list of additional inputs found by dependency discovery, during action preparation, + * and clears the stored list. {@link #prepare} must be called before this method is called, on + * each action execution. + */ + public Collection<? extends ActionInput> getAdditionalInputs() { + Collection<? extends ActionInput> result = Preconditions.checkNotNull(additionalInputs); + additionalInputs = null; + return result; + } + + @Override + public boolean discoversInputs() { + return true; + } + + @Override + public void discoverInputs(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + try { + this.additionalInputs = executor.getContext(CppCompileActionContext.class) + .findAdditionalInputs(this, actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException("Include scanning of rule '" + getOwner().getLabel() + "'", + executor.getVerboseFailures(), this); + } + } + + @Override + public Artifact getPrimaryInput() { + return getSourceFile(); + } + + @Override + public Artifact getPrimaryOutput() { + return getOutputFile(); + } + + /** + * Returns the path of the c/cc source for gcc. + */ + public final Artifact getSourceFile() { + return cppCompileCommandLine.sourceFile; + } + + /** + * Returns the path where gcc should put its result. + */ + public Artifact getOutputFile() { + return outputFile; + } + + /** + * Returns the path of the debug info output file (when debug info is + * spliced out of the .o file via fission). + */ + @Nullable + Artifact getDwoFile() { + return dwoFile; + } + + protected PathFragment getInternalOutputFile() { + return outputFile.getExecPath(); + } + + @VisibleForTesting + public List<String> getPluginOpts() { + return cppCompileCommandLine.pluginOpts; + } + + Collection<PathFragment> getExtraSystemIncludePrefixes() { + return extraSystemIncludePrefixes; + } + + @Override + public Map<Artifact, Path> getLegalGeneratedScannerFileMap() { + Map<Artifact, Path> legalOuts = new HashMap<>(); + + for (Artifact a : context.getDeclaredIncludeSrcs()) { + if (!a.isSourceArtifact()) { + legalOuts.put(a, null); + } + } + for (Pair<Artifact, Artifact> pregreppedSrcs : context.getPregreppedHeaders()) { + Artifact hdr = pregreppedSrcs.getFirst(); + Preconditions.checkState(!hdr.isSourceArtifact(), hdr); + legalOuts.put(hdr, pregreppedSrcs.getSecond().getPath()); + } + return Collections.unmodifiableMap(legalOuts); + } + + /** + * Returns the path where gcc should put the discovered dependency + * information. + */ + public DotdFile getDotdFile() { + return cppCompileCommandLine.dotdFile; + } + + protected boolean needsIncludeScanning(Executor executor) { + return executor.getContext(actionContext).needsIncludeScanning(); + } + + @Override + public String describeStrategy(Executor executor) { + return executor.getContext(actionContext).strategyLocality(); + } + + @VisibleForTesting + public CppCompilationContext getContext() { + return context; + } + + @Override + public List<PathFragment> getQuoteIncludeDirs() { + return context.getQuoteIncludeDirs(); + } + + @Override + public List<PathFragment> getIncludeDirs() { + ImmutableList.Builder<PathFragment> result = ImmutableList.builder(); + result.addAll(context.getIncludeDirs()); + for (String opt : cppCompileCommandLine.copts) { + if (opt.startsWith("-I") && opt.length() > 2) { + // We insist on the combined form "-Idir". + result.add(new PathFragment(opt.substring(2))); + } + } + return result.build(); + } + + @Override + public List<PathFragment> getSystemIncludeDirs() { + ImmutableList.Builder<PathFragment> result = ImmutableList.builder(); + result.addAll(context.getSystemIncludeDirs()); + for (String opt : cppCompileCommandLine.copts) { + if (opt.startsWith("-isystem") && opt.length() > 8) { + // We insist on the combined form "-isystemdir". + result.add(new PathFragment(opt.substring(8))); + } + } + return result.build(); + } + + @Override + public List<String> getCmdlineIncludes() { + ImmutableList.Builder<String> cmdlineIncludes = ImmutableList.builder(); + List<String> args = getArgv(); + for (Iterator<String> argi = args.iterator(); argi.hasNext();) { + String arg = argi.next(); + if (arg.equals("-include") && argi.hasNext()) { + cmdlineIncludes.add(argi.next()); + } + } + return cmdlineIncludes.build(); + } + + @Override + public Collection<Artifact> getIncludeScannerSources() { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + // For every header module we use for the build we need the set of sources that it can + // reference. + builder.addAll(context.getTransitiveHeaderModuleSrcs()); + if (CppFileTypes.CPP_MODULE_MAP.matches(getSourceFile().getPath())) { + // If this is an action that compiles the header module itself, the source we build is the + // module map, and we need to include-scan all headers that are referenced in the module map. + // We need to do include scanning as long as we want to support building code bases that are + // not fully strict layering clean. + builder.addAll(context.getHeaderModuleSrcs()); + } else { + builder.add(getSourceFile()); + } + return builder.build().toCollection(); + } + + @Override + public Iterable<IncludeScannable> getAuxiliaryScannables() { + return lipoScannables; + } + + /** + * Returns the list of "-D" arguments that should be used by this gcc + * invocation. Only used for testing. + */ + @VisibleForTesting + public ImmutableCollection<String> getDefines() { + return context.getDefines(); + } + + /** + * Returns an (immutable) map of environment key, value pairs to be + * provided to the C++ compiler. + */ + public ImmutableMap<String, String> getEnvironment() { + Map<String, String> environment = + new LinkedHashMap<>(configuration.getDefaultShellEnvironment()); + if (configuration.isCodeCoverageEnabled()) { + environment.put("PWD", "/proc/self/cwd"); + } + if (OS.getCurrent() == OS.WINDOWS) { + // TODO(bazel-team): Both GCC and clang rely on their execution directories being on + // PATH, otherwise they fail to find dependent DLLs (and they fail silently...). On + // the other hand, Windows documentation says that the directory of the executable + // is always searched for DLLs first. Not sure what to make of it. + // Other options are to forward the system path (brittle), or to add a PATH field to + // the crosstool file. + environment.put("PATH", cppConfiguration.getToolPathFragment(Tool.GCC).getParentDirectory() + .getPathString()); + } + return ImmutableMap.copyOf(environment); + } + + /** + * Returns a new, mutable list of command and arguments (argv) to be passed + * to the gcc subprocess. + */ + public final List<String> getArgv() { + return getArgv(getInternalOutputFile()); + } + + protected final List<String> getArgv(PathFragment outputFile) { + return cppCompileCommandLine.getArgv(outputFile); + } + + @Override + public ExtraActionInfo.Builder getExtraActionInfo() { + CppCompileInfo.Builder info = CppCompileInfo.newBuilder(); + info.setTool(cppConfiguration.getToolPathFragment(Tool.GCC).getPathString()); + for (String option : getCompilerOptions()) { + info.addCompilerOption(option); + } + info.setOutputFile(outputFile.getExecPathString()); + info.setSourceFile(getSourceFile().getExecPathString()); + if (inputsKnown()) { + info.addAllSourcesAndHeaders(Artifact.toExecPaths(getInputs())); + } else { + info.addSourcesAndHeaders(getSourceFile().getExecPathString()); + info.addAllSourcesAndHeaders( + Artifact.toExecPaths(context.getDeclaredIncludeSrcs())); + } + + return super.getExtraActionInfo() + .setExtension(CppCompileInfo.cppCompileInfo, info.build()); + } + + /** + * Returns the compiler options. + */ + @VisibleForTesting + public List<String> getCompilerOptions() { + return cppCompileCommandLine.getCompilerOptions(); + } + + /** + * Enforce that the includes actually visited during the compile were properly + * declared in the rules. + * + * <p>The technique is to walk through all of the reported includes that gcc + * emits into the .d file, and verify that they came from acceptable + * relative include directories. This is done in two steps: + * + * <p>First, each included file is stripped of any include path prefix from + * {@code quoteIncludeDirs} to produce an effective relative include dir+name. + * + * <p>Second, the remaining directory is looked up in {@code declaredIncludeDirs}, + * a list of acceptable dirs. This list contains a set of dir fragments that + * have been calculated by the configured target to be allowable for inclusion + * by this source. If no match is found, an error is reported and an exception + * is thrown. + * + * @throws ActionExecutionException iff there was an undeclared dependency + */ + @VisibleForTesting + public void validateInclusions( + MiddlemanExpander middlemanExpander, EventHandler eventHandler) + throws ActionExecutionException { + if (!cppConfiguration.shouldScanIncludes() || !inputsKnown()) { + return; + } + + IncludeProblems errors = new IncludeProblems(); + IncludeProblems warnings = new IncludeProblems(); + Set<Artifact> allowedIncludes = new HashSet<>(); + for (Artifact input : mandatoryInputs) { + if (input.isMiddlemanArtifact()) { + middlemanExpander.expand(input, allowedIncludes); + } + allowedIncludes.add(input); + } + + if (optionalSourceFile != null) { + allowedIncludes.add(optionalSourceFile); + } + List<PathFragment> cxxSystemIncludeDirs = + cppConfiguration.getBuiltInIncludeDirectories(); + Iterable<PathFragment> ignoreDirs = Iterables.concat(cxxSystemIncludeDirs, + extraSystemIncludePrefixes, context.getSystemIncludeDirs()); + + // Copy the sets to hash sets for fast contains checking. + // Avoid immutable sets here to limit memory churn. + Set<PathFragment> declaredIncludeDirs = Sets.newHashSet(context.getDeclaredIncludeDirs()); + Set<PathFragment> warnIncludeDirs = Sets.newHashSet(context.getDeclaredIncludeWarnDirs()); + Set<Artifact> declaredIncludeSrcs = Sets.newHashSet(context.getDeclaredIncludeSrcs()); + for (Artifact input : getInputs()) { + if (context.getCompilationPrerequisites().contains(input) + || allowedIncludes.contains(input)) { + continue; // ignore our fixed source in mandatoryInput: we just want includes + } + // Ignore headers from built-in include directories. + if (FileSystemUtils.startsWithAny(input.getExecPath(), ignoreDirs)) { + continue; + } + if (!isDeclaredIn(input, declaredIncludeDirs, declaredIncludeSrcs)) { + // This call can never match the declared include sources (they would be matched above). + // There are no declared include sources we need to warn about, so use an empty set here. + if (isDeclaredIn(input, warnIncludeDirs, ImmutableSet.<Artifact>of())) { + warnings.add(input.getPath().toString()); + } else { + errors.add(input.getPath().toString()); + } + } + } + if (VALIDATION_DEBUG_WARN) { + synchronized (System.err) { + if (VALIDATION_DEBUG >= 2 || errors.hasProblems() || warnings.hasProblems()) { + if (errors.hasProblems()) { + System.err.println("ERROR: Include(s) were not in declared srcs:"); + } else if (warnings.hasProblems()) { + System.err.println("WARN: Include(s) were not in declared srcs:"); + } else { + System.err.println("INFO: Include(s) were OK for '" + getSourceFile() + + "', declared srcs:"); + } + for (Artifact a : context.getDeclaredIncludeSrcs()) { + System.err.println(" '" + a.toDetailString() + "'"); + } + System.err.println(" or under declared dirs:"); + for (PathFragment f : Sets.newTreeSet(context.getDeclaredIncludeDirs())) { + System.err.println(" '" + f + "'"); + } + System.err.println(" or under declared warn dirs:"); + for (PathFragment f : Sets.newTreeSet(context.getDeclaredIncludeWarnDirs())) { + System.err.println(" '" + f + "'"); + } + System.err.println(" with prefixes:"); + for (PathFragment dirpath : context.getQuoteIncludeDirs()) { + System.err.println(" '" + dirpath + "'"); + } + } + } + } + + if (warnings.hasProblems()) { + eventHandler.handle( + new Event(EventKind.WARNING, + getOwner().getLocation(), warnings.getMessage(this, getSourceFile()), + Label.print(getOwner().getLabel()))); + } + errors.assertProblemFree(this, getSourceFile()); + } + + /** + * Returns true if an included artifact is declared in a set of allowed + * include directories. The simple case is that the artifact's parent + * directory is contained in the set, or is empty. + * + * <p>This check also supports a wildcard suffix of '**' for the cases where the + * calculations are inexact. + * + * <p>It also handles unseen non-nested-package subdirs by walking up the path looking + * for matches. + */ + private static boolean isDeclaredIn(Artifact input, Set<PathFragment> declaredIncludeDirs, + Set<Artifact> declaredIncludeSrcs) { + // First check if it's listed in "srcs". If so, then its declared & OK. + if (declaredIncludeSrcs.contains(input)) { + return true; + } + // If it's a derived artifact, then it MUST be listed in "srcs" as checked above. + // We define derived here as being not source and not under the include link tree. + if (!input.isSourceArtifact() + && !input.getRoot().getExecPath().getBaseName().equals("include")) { + return false; + } + // Need to do dir/package matching: first try a quick exact lookup. + PathFragment includeDir = input.getRootRelativePath().getParentDirectory(); + if (includeDir.segmentCount() == 0 || declaredIncludeDirs.contains(includeDir)) { + return true; // OK: quick exact match. + } + // Not found in the quick lookup: try the wildcards. + for (PathFragment declared : declaredIncludeDirs) { + if (declared.getBaseName().equals("**")) { + if (includeDir.startsWith(declared.getParentDirectory())) { + return true; // OK: under a wildcard dir. + } + } + } + // Still not found: see if it is in a subdir of a declared package. + Path root = input.getRoot().getPath(); + for (Path dir = input.getPath().getParentDirectory();;) { + if (dir.getRelative("BUILD").exists()) { + return false; // Bad: this is a sub-package, not a subdir of a declared package. + } + dir = dir.getParentDirectory(); + if (dir.equals(root)) { + return false; // Bad: at the top, give up. + } + if (declaredIncludeDirs.contains(dir.relativeTo(root))) { + return true; // OK: found under a declared dir. + } + } + } + + /** + * Recalculates this action's live input collection, including sources, middlemen. + * + * @throws ActionExecutionException iff any errors happen during update. + */ + @VisibleForTesting + @ThreadCompatible + public final void updateActionInputs(Path execRoot, + ArtifactResolver artifactResolver, CppCompileActionContext.Reply reply) + throws ActionExecutionException { + if (!cppConfiguration.shouldScanIncludes()) { + return; + } + inputsKnown = false; + NestedSetBuilder<Artifact> inputs = NestedSetBuilder.stableOrder(); + Profiler.instance().startTask(ProfilerTask.ACTION_UPDATE, this); + try { + inputs.addTransitive(mandatoryInputs); + if (optionalSourceFile != null) { + inputs.add(optionalSourceFile); + } + inputs.addAll(context.getCompilationPrerequisites()); + populateActionInputs(execRoot, artifactResolver, reply, inputs); + inputsKnown = true; + } finally { + Profiler.instance().completeTask(ProfilerTask.ACTION_UPDATE); + synchronized (this) { + setInputs(inputs.build()); + } + } + } + + private DependencySet processDepset(Path execRoot, CppCompileActionContext.Reply reply) + throws IOException { + DependencySet depSet = new DependencySet(execRoot); + + // artifact() is null if we are not using in-memory .d files. We also want to prepare for the + // case where we expected an in-memory .d file, but we did not get an appropriate response. + // Perhaps we produced the file locally. + if (getDotdFile().artifact() != null || reply == null) { + return depSet.read(getDotdFile().getPath()); + } else { + // This is an in-memory .d file. + return depSet.process(reply.getContents()); + } + } + + /** + * Populates the given ordered collection with additional input artifacts + * relevant to the specific action implementation. + * + * <p>The default implementation updates this Action's input set by reading + * dynamically-discovered dependency information out of the .d file. + * + * <p>Artifacts are considered inputs but not "mandatory" inputs. + * + * + * @param reply the reply from the compilation. + * @param inputs the ordered collection of inputs to append to + * @throws ActionExecutionException iff the .d is missing, malformed or has + * unresolvable included artifacts. + */ + @ThreadCompatible + private void populateActionInputs(Path execRoot, + ArtifactResolver artifactResolver, CppCompileActionContext.Reply reply, + NestedSetBuilder<Artifact> inputs) + throws ActionExecutionException { + try { + // Read .d file. + DependencySet depSet = processDepset(execRoot, reply); + + // Determine prefixes of allowed absolute inclusions. + CppConfiguration toolchain = cppConfiguration; + List<PathFragment> systemIncludePrefixes = new ArrayList<>(); + for (PathFragment includePath : toolchain.getBuiltInIncludeDirectories()) { + if (includePath.isAbsolute()) { + systemIncludePrefixes.add(includePath); + } + } + systemIncludePrefixes.addAll(extraSystemIncludePrefixes); + + // Check inclusions. + IncludeProblems problems = new IncludeProblems(); + Map<PathFragment, Artifact> allowedDerivedInputsMap = getAllowedDerivedInputsMap(); + for (PathFragment execPath : depSet.getDependencies()) { + if (execPath.isAbsolute()) { + // Absolute includes from system paths are ignored. + if (FileSystemUtils.startsWithAny(execPath, systemIncludePrefixes)) { + continue; + } + // Since gcc is given only relative paths on the command line, + // non-system include paths here should never be absolute. If they + // are, it's probably due to a non-hermetic #include, & we should stop + // the build with an error. + if (execPath.startsWith(execRoot.asFragment())) { + execPath = execPath.relativeTo(execRoot.asFragment()); // funky but tolerable path + } else { + problems.add(execPath.getPathString()); + continue; + } + } + Artifact artifact = allowedDerivedInputsMap.get(execPath); + if (artifact == null) { + artifact = artifactResolver.resolveSourceArtifact(execPath); + } + if (artifact != null) { + inputs.add(artifact); + // In some cases, execution backends need extra files for each included file. Add them + // to the set of actual inputs. + inputs.addAll(includeResolver.getInputsForIncludedFile(artifact, artifactResolver)); + } else { + // Abort if we see files that we can't resolve, likely caused by + // undeclared includes or illegal include constructs. + problems.add(execPath.getPathString()); + } + } + problems.assertProblemFree(this, getSourceFile()); + } catch (IOException e) { + // Some kind of IO or parse exception--wrap & rethrow it to stop the build. + throw new ActionExecutionException("error while parsing .d file", e, this, false); + } + } + + @Override + public void updateInputsFromCache( + ArtifactResolver artifactResolver, Collection<PathFragment> inputPaths) { + // Note that this method may trigger a violation of the desirable invariant that getInputs() + // is a superset of getMandatoryInputs(). See bug about an "action not in canonical form" + // error message and the integration test test_crosstool_change_and_failure(). + + Map<PathFragment, Artifact> allowedDerivedInputsMap = getAllowedDerivedInputsMap(); + List<Artifact> inputs = new ArrayList<>(); + for (PathFragment execPath : inputPaths) { + // The artifact may be a derived artifact, and if it has been created already, then we still + // want to keep it to preserve incrementality. + Artifact artifact = allowedDerivedInputsMap.get(execPath); + if (artifact == null) { + artifact = artifactResolver.resolveSourceArtifact(execPath); + } + // If PathFragment cannot be resolved into the artifact - ignore it. This could happen if + // rule definition has changed and action no longer depends on, e.g., additional source file + // in the separate package and that package is no longer referenced anywhere else. + // It is safe to ignore such paths because dependency checker would identify change in inputs + // (ignored path was used before) and will force action execution. + if (artifact != null) { + inputs.add(artifact); + } + } + inputsKnown = true; + synchronized (this) { + setInputs(inputs); + } + } + + private Map<PathFragment, Artifact> getAllowedDerivedInputsMap() { + Map<PathFragment, Artifact> allowedDerivedInputMap = new HashMap<>(); + addToMap(allowedDerivedInputMap, mandatoryInputs); + addToMap(allowedDerivedInputMap, context.getDeclaredIncludeSrcs()); + addToMap(allowedDerivedInputMap, context.getCompilationPrerequisites()); + Artifact artifact = getSourceFile(); + if (!artifact.isSourceArtifact()) { + allowedDerivedInputMap.put(artifact.getExecPath(), artifact); + } + return allowedDerivedInputMap; + } + + private void addToMap(Map<PathFragment, Artifact> map, Iterable<Artifact> artifacts) { + for (Artifact artifact : artifacts) { + if (!artifact.isSourceArtifact()) { + map.put(artifact.getExecPath(), artifact); + } + } + } + + @Override + protected String getRawProgressMessage() { + return "Compiling " + getSourceFile().prettyPrint(); + } + + /** + * Return the directories in which to look for headers (pertains to headers + * not specifically listed in {@code declaredIncludeSrcs}). The return value + * may contain duplicate elements. + */ + public NestedSet<PathFragment> getDeclaredIncludeDirs() { + return context.getDeclaredIncludeDirs(); + } + + /** + * Return the directories in which to look for headers and issue a warning. + * (pertains to headers not specifically listed in {@code + * declaredIncludeSrcs}). The return value may contain duplicate elements. + */ + public NestedSet<PathFragment> getDeclaredIncludeWarnDirs() { + return context.getDeclaredIncludeWarnDirs(); + } + + /** + * Return explicit header files (i.e., header files explicitly listed). The + * return value may contain duplicate elements. + */ + public NestedSet<Artifact> getDeclaredIncludeSrcs() { + return context.getDeclaredIncludeSrcs(); + } + + /** + * Return explicit header files (i.e., header files explicitly listed) in an order + * that is stable between builds. + */ + protected final List<PathFragment> getDeclaredIncludeSrcsInStableOrder() { + List<PathFragment> paths = new ArrayList<>(); + for (Artifact declaredIncludeSrc : context.getDeclaredIncludeSrcs()) { + paths.add(declaredIncludeSrc.getExecPath()); + } + Collections.sort(paths); // Order is not important, but stability is. + return paths; + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return executor.getContext(actionContext).estimateResourceConsumption(this); + } + + @VisibleForTesting + public Class<? extends CppCompileActionContext> getActionContext() { + return actionContext; + } + + /** + * Estimate resource consumption when this action is executed locally. + */ + public ResourceSet estimateResourceConsumptionLocal() { + // We use a local compile, so much of the time is spent waiting for IO, + // but there is still significant CPU; hence we estimate 50% cpu usage. + return new ResourceSet(/*memoryMb=*/200, /*cpuUsage=*/0.5, /*ioUsage=*/0.0); + } + + @Override + public String computeKey() { + Fingerprint f = new Fingerprint(); + f.addUUID(actionClassId); + f.addStrings(getArgv()); + + /* + * getArgv() above captures all changes which affect the compilation + * command and hence the contents of the object file. But we need to + * also make sure that we reexecute the action if any of the fields + * that affect whether validateIncludes() will report an error or warning + * have changed, otherwise we might miss some errors. + */ + f.addPaths(context.getDeclaredIncludeDirs()); + f.addPaths(context.getDeclaredIncludeWarnDirs()); + f.addPaths(getDeclaredIncludeSrcsInStableOrder()); + f.addPaths(getExtraSystemIncludePrefixes()); + return f.hexDigestAndReset(); + } + + @Override + @ThreadCompatible + public void execute( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + CppCompileActionContext.Reply reply; + try { + reply = executor.getContext(actionContext).execWithReply(this, actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException("C++ compilation of rule '" + getOwner().getLabel() + "'", + executor.getVerboseFailures(), this); + } + ensureCoverageNotesFilesExist(); + IncludeScanningContext scanningContext = executor.getContext(IncludeScanningContext.class); + updateActionInputs(executor.getExecRoot(), scanningContext.getArtifactResolver(), reply); + reply = null; // Clear in-memory .d files early. + validateInclusions(actionExecutionContext.getMiddlemanExpander(), executor.getEventHandler()); + } + + /** + * Gcc only creates ".gcno" files if the compilation unit is non-empty. + * To ensure that the set of outputs for a CppCompileAction remains consistent + * and doesn't vary dynamically depending on the _contents_ of the input files, + * we create empty ".gcno" files if gcc didn't create them. + */ + private void ensureCoverageNotesFilesExist() throws ActionExecutionException { + for (Artifact output : getOutputs()) { + if (CppFileTypes.COVERAGE_NOTES.matches(output.getFilename()) // ".gcno" + && !output.getPath().exists()) { + try { + FileSystemUtils.createEmptyFile(output.getPath()); + } catch (IOException e) { + throw new ActionExecutionException( + "Error creating file '" + output.getPath() + "': " + e.getMessage(), e, this, false); + } + } + } + } + + /** + * Provides list of include files needed for performing extra actions on this action when run + * remotely. The list of include files is created by performing a header scan on the known input + * files. + */ + @Override + public Iterable<Artifact> getInputFilesForExtraAction( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Collection<Artifact> scannedIncludes = + actionExecutionContext.getExecutor().getContext(actionContext) + .getScannedIncludeFiles(this, actionExecutionContext); + // Use a set to eliminate duplicates. + ImmutableSet.Builder<Artifact> result = ImmutableSet.builder(); + return result.addAll(getInputs()).addAll(scannedIncludes).build(); + } + + @Override + public String getMnemonic() { return "CppCompile"; } + + @Override + public String describeKey() { + StringBuilder message = new StringBuilder(); + message.append(getProgressMessage()); + message.append('\n'); + message.append(" Command: "); + message.append( + ShellEscaper.escapeString(cppConfiguration.getLdExecutable().getPathString())); + message.append('\n'); + // Outputting one argument per line makes it easier to diff the results. + for (String argument : ShellEscaper.escapeAll(getArgv())) { + message.append(" Argument: "); + message.append(argument); + message.append('\n'); + } + + for (PathFragment path : context.getDeclaredIncludeDirs()) { + message.append(" Declared include directory: "); + message.append(ShellEscaper.escapeString(path.getPathString())); + message.append('\n'); + } + + for (PathFragment path : getDeclaredIncludeSrcsInStableOrder()) { + message.append(" Declared include source: "); + message.append(ShellEscaper.escapeString(path.getPathString())); + message.append('\n'); + } + + for (PathFragment path : getExtraSystemIncludePrefixes()) { + message.append(" Extra system include prefix: "); + message.append(ShellEscaper.escapeString(path.getPathString())); + message.append('\n'); + } + return message.toString(); + } + + /** + * The compile command line for the enclosing C++ compile action. + */ + public final class CppCompileCommandLine { + private final Artifact sourceFile; + private final DotdFile dotdFile; + private final CppModuleMap cppModuleMap; + private final List<String> copts; + private final Predicate<String> coptsFilter; + private final List<String> pluginOpts; + private final boolean isInstrumented; + private final Collection<String> features; + private final FeatureConfiguration featureConfiguration; + + // The value of the BUILD_FDO_TYPE macro to be defined on command line + @Nullable private final String fdoBuildStamp; + + public CppCompileCommandLine(Artifact sourceFile, DotdFile dotdFile, CppModuleMap cppModuleMap, + ImmutableList<String> copts, Predicate<String> coptsFilter, + ImmutableList<String> pluginOpts, boolean isInstrumented, + Collection<String> features, FeatureConfiguration featureConfiguration, + @Nullable String fdoBuildStamp) { + this.sourceFile = Preconditions.checkNotNull(sourceFile); + this.dotdFile = Preconditions.checkNotNull(dotdFile); + this.cppModuleMap = cppModuleMap; + this.copts = Preconditions.checkNotNull(copts); + this.coptsFilter = coptsFilter; + this.pluginOpts = Preconditions.checkNotNull(pluginOpts); + this.isInstrumented = isInstrumented; + this.features = Preconditions.checkNotNull(features); + this.featureConfiguration = featureConfiguration; + this.fdoBuildStamp = fdoBuildStamp; + } + + protected List<String> getArgv(PathFragment outputFile) { + List<String> commandLine = new ArrayList<>(); + + // first: The command name. + commandLine.add(cppConfiguration.getToolPathFragment(Tool.GCC).getPathString()); + + // second: The compiler options. + commandLine.addAll(getCompilerOptions()); + + // third: The file to compile! + commandLine.add("-c"); + commandLine.add(sourceFile.getExecPathString()); + + // finally: The output file. (Prefixed with -o). + commandLine.add("-o"); + commandLine.add(outputFile.getPathString()); + + return commandLine; + } + + private String getActionName() { + PathFragment sourcePath = sourceFile.getExecPath(); + if (CppFileTypes.CPP_MODULE_MAP.matches(sourcePath)) { + return CPP_MODULE_COMPILE; + } else if (CppFileTypes.CPP_HEADER.matches(sourcePath)) { + // TODO(bazel-team): Handle C headers that probably don't work in C++ mode. + // TODO(bazel-team): Replace use of features.contains with featureConfiguration.isEnabled + // here (the other instances will need to stay with the current feature selection process + // until all crosstool configurations have been converted). + if (features.contains(CppRuleClasses.PARSE_HEADERS)) { + return CPP_HEADER_PARSING; + } else if (features.contains(CppRuleClasses.PREPROCESS_HEADERS)) { + return CPP_HEADER_PREPROCESSING; + } else { + // CcCommon.collectCAndCppSources() ensures we do not add headers to + // the compilation artifacts unless either 'parse_headers' or + // 'preprocess_headers' is set. + throw new IllegalStateException(); + } + } else if (CppFileTypes.C_SOURCE.matches(sourcePath)) { + return C_COMPILE; + } else if (CppFileTypes.CPP_SOURCE.matches(sourcePath)) { + return CPP_COMPILE; + } else if (CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR.matches(sourcePath)) { + return PREPROCESS_ASSEMBLE; + } + // CcLibraryHelper ensures CppCompileAction only gets instantiated for supported file types. + throw new IllegalStateException(); + } + + public List<String> getCompilerOptions() { + List<String> options = new ArrayList<>(); + + // TODO(bazel-team): Extract combinations of options into sections in the CROSSTOOL file. + if (CppFileTypes.CPP_MODULE_MAP.matches(sourceFile.getExecPath())) { + options.add("-x"); + options.add("c++"); + } else if (CppFileTypes.CPP_HEADER.matches(sourceFile.getExecPath())) { + // TODO(bazel-team): Read the compiler flag settings out of the CROSSTOOL file. + // TODO(bazel-team): Handle C headers that probably don't work in C++ mode. + if (features.contains(CppRuleClasses.PARSE_HEADERS)) { + options.add("-x"); + options.add("c++-header"); + // Specifying -x c++-header will make clang/gcc create precompiled + // headers, which we suppress with -fsyntax-only. + options.add("-fsyntax-only"); + } else if (features.contains(CppRuleClasses.PREPROCESS_HEADERS)) { + options.add("-E"); + options.add("-x"); + options.add("c++"); + } else { + // CcCommon.collectCAndCppSources() ensures we do not add headers to + // the compilation artifacts unless either 'parse_headers' or + // 'preprocess_headers' is set. + throw new IllegalStateException(); + } + } + + for (PathFragment quoteIncludePath : context.getQuoteIncludeDirs()) { + // "-iquote" is a gcc-specific option. For C compilers that don't support "-iquote", + // we should instead use "-I". + options.add("-iquote"); + options.add(quoteIncludePath.getSafePathString()); + } + for (PathFragment includePath : context.getIncludeDirs()) { + options.add("-I" + includePath.getSafePathString()); + } + for (PathFragment systemIncludePath : context.getSystemIncludeDirs()) { + options.add("-isystem"); + options.add(systemIncludePath.getSafePathString()); + } + + CppConfiguration toolchain = cppConfiguration; + + // pluginOpts has to be added before defaultCopts because -fplugin must precede -plugin-arg. + options.addAll(pluginOpts); + addFilteredOptions(options, toolchain.getCompilerOptions(features)); + + // Enable instrumentation if requested. + if (isInstrumented) { + addFilteredOptions(options, ImmutableList.of("-fprofile-arcs", "-ftest-coverage")); + } + + String sourceFilename = sourceFile.getExecPathString(); + if (CppFileTypes.C_SOURCE.matches(sourceFilename)) { + addFilteredOptions(options, toolchain.getCOptions()); + } + if (CppFileTypes.CPP_SOURCE.matches(sourceFilename) + || CppFileTypes.CPP_HEADER.matches(sourceFilename) + || CppFileTypes.CPP_MODULE_MAP.matches(sourceFilename)) { + addFilteredOptions(options, toolchain.getCxxOptions(features)); + } + + // Users don't expect the explicit copts to be filtered by coptsFilter, add them verbatim. + options.addAll(copts); + + for (String warn : cppConfiguration.getCWarns()) { + options.add("-W" + warn); + } + for (String define : context.getDefines()) { + options.add("-D" + define); + } + + // Stamp FDO builds with FDO subtype string + if (fdoBuildStamp != null) { + options.add("-D" + CppConfiguration.FDO_STAMP_MACRO + "=\"" + fdoBuildStamp + "\""); + } + + options.addAll(toolchain.getUnfilteredCompilerOptions(features)); + + // GCC gives randomized names to symbols which are defined in + // an anonymous namespace but have external linkage. To make + // computation of these deterministic, we want to override the + // default seed for the random number generator. It's safe to use + // any value which differs for all translation units; we use the + // path to the object file. + options.add("-frandom-seed=" + outputFile.getExecPathString()); + + // Add the options of --per_file_copt, if the label or the base name of the source file + // matches the specified regular expression filter. + for (PerLabelOptions perLabelOptions : cppConfiguration.getPerFileCopts()) { + if ((sourceLabel != null && perLabelOptions.isIncluded(sourceLabel)) + || perLabelOptions.isIncluded(sourceFile)) { + options.addAll(perLabelOptions.getOptions()); + } + } + + // Enable <object>.d file generation. + if (dotdFile != null) { + // Gcc options: + // -MD turns on .d file output as a side-effect (doesn't imply -E) + // -MM[D] enables user includes only, not system includes + // -MF <name> specifies the dotd file name + // Issues: + // -M[M] alone subverts actual .o output (implies -E) + // -M[M]D alone breaks some of the .d naming assumptions + // This combination gets user and system includes with specified name: + // -MD -MF <name> + options.add("-MD"); + options.add("-MF"); + options.add(dotdFile.getSafeExecPath().getPathString()); + } + + if (cppModuleMap != null && (compileHeaderModules || enableLayeringCheck)) { + options.add("-Xclang-only=-fmodule-maps"); + options.add("-Xclang-only=-fmodule-name=" + cppModuleMap.getName()); + options.add("-Xclang-only=-fmodule-map-file=" + + cppModuleMap.getArtifact().getExecPathString()); + options.add("-Xclang=-fno-modules-implicit-maps"); + + if (compileHeaderModules) { + options.add("-Xclang-only=-fmodules"); + if (CppFileTypes.CPP_MODULE_MAP.matches(sourceFilename)) { + options.add("-Xclang=-emit-module"); + options.add("-Xcrosstool-module-compilation"); + } + // Select .pcm inputs to pass on the command line depending on whether + // we are in pic or non-pic mode. + // TODO(bazel-team): We want to add these to the compile even if the + // current target is not built as a module; currently that implies + // passing -fmodules to the compiler, which is experimental; thus, we + // do not use the header modules files for now if the current + // compilation is not modules enabled on its own. + boolean pic = copts.contains("-fPIC"); + for (Artifact source : context.getAdditionalInputs()) { + if ((pic && source.getFilename().endsWith(".pic.pcm")) || (!pic + && !source.getFilename().endsWith(".pic.pcm") + && source.getFilename().endsWith(".pcm"))) { + options.add("-Xclang=-fmodule-file=" + source.getExecPathString()); + } + } + } + if (enableLayeringCheck) { + options.add("-Xclang-only=-fmodules-strict-decluse"); + } + } + + if (FileType.contains(outputFile, CppFileTypes.ASSEMBLER, CppFileTypes.PIC_ASSEMBLER)) { + options.add("-S"); + } else if (FileType.contains(outputFile, CppFileTypes.PREPROCESSED_C, + CppFileTypes.PREPROCESSED_CPP, CppFileTypes.PIC_PREPROCESSED_C, + CppFileTypes.PIC_PREPROCESSED_CPP)) { + options.add("-E"); + } + + if (cppConfiguration.useFission()) { + options.add("-gsplit-dwarf"); + } + + options.addAll(featureConfiguration.getCommandLine(getActionName(), + ImmutableMultimap.<String, String>of())); + return options; + } + + // For each option in 'in', add it to 'out' unless it is matched by the 'coptsFilter' regexp. + private void addFilteredOptions(List<String> out, List<String> in) { + Iterables.addAll(out, Iterables.filter(in, coptsFilter)); + } + } + + /** + * A reference to a .d file. There are two modes: + * <ol> + * <li>an Artifact that represents a real on-disk file + * <li>just an execPath that refers to a virtual .d file that is not written to disk + * </ol> + */ + public static class DotdFile { + private final Artifact artifact; + private final PathFragment execPath; + + public DotdFile(Artifact artifact) { + this.artifact = artifact; + this.execPath = null; + } + + public DotdFile(PathFragment execPath) { + this.artifact = null; + this.execPath = execPath; + } + + /** + * @return the Artifact or null + */ + public Artifact artifact() { + return artifact; + } + + /** + * @return Gets the execPath regardless of whether this is a real Artifact + */ + public PathFragment getSafeExecPath() { + return execPath == null ? artifact.getExecPath() : execPath; + } + + /** + * @return the on-disk location of the .d file or null + */ + public Path getPath() { + return artifact.getPath(); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java new file mode 100644 index 0000000000..0d8da3e4a2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java @@ -0,0 +1,439 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Functions; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppCompileAction.DotdFile; +import com.google.devtools.build.lib.rules.cpp.CppCompileAction.IncludeResolver; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.UUID; +import java.util.regex.Pattern; + +/** + * Builder class to construct C++ compile actions. + */ +public class CppCompileActionBuilder { + public static final UUID GUID = UUID.fromString("cee5db0a-d2ad-4c69-9b81-97c936a29075"); + + private final ActionOwner owner; + private final List<String> features = new ArrayList<>(); + private CcToolchainFeatures.FeatureConfiguration featureConfiguration; + private final Artifact sourceFile; + private final Label sourceLabel; + private final NestedSetBuilder<Artifact> mandatoryInputsBuilder; + private NestedSetBuilder<Artifact> pluginInputsBuilder; + private Artifact optionalSourceFile; + private Artifact outputFile; + private PathFragment tempOutputFile; + private DotdFile dotdFile; + private Artifact gcnoFile; + private final BuildConfiguration configuration; + private CppCompilationContext context = CppCompilationContext.EMPTY; + private final List<String> copts = new ArrayList<>(); + private final List<String> pluginOpts = new ArrayList<>(); + private final List<Pattern> nocopts = new ArrayList<>(); + private AnalysisEnvironment analysisEnvironment; + private ImmutableList<PathFragment> extraSystemIncludePrefixes = ImmutableList.of(); + private boolean enableLayeringCheck; + private boolean compileHeaderModules; + private String fdoBuildStamp; + private IncludeResolver includeResolver = CppCompileAction.VOID_INCLUDE_RESOLVER; + private UUID actionClassId = GUID; + private Class<? extends CppCompileActionContext> actionContext; + private CppConfiguration cppConfiguration; + private ImmutableMap<Artifact, IncludeScannable> lipoScannableMap; + + /** + * Creates a builder from a rule. This also uses the configuration and + * artifact factory from the rule. + */ + public CppCompileActionBuilder(RuleContext ruleContext, Artifact sourceFile, Label sourceLabel) { + this.owner = ruleContext.getActionOwner(); + this.actionContext = CppCompileActionContext.class; + this.cppConfiguration = ruleContext.getFragment(CppConfiguration.class); + this.analysisEnvironment = ruleContext.getAnalysisEnvironment(); + this.sourceFile = sourceFile; + this.sourceLabel = sourceLabel; + this.configuration = ruleContext.getConfiguration(); + this.mandatoryInputsBuilder = NestedSetBuilder.stableOrder(); + this.pluginInputsBuilder = NestedSetBuilder.stableOrder(); + this.lipoScannableMap = getLipoScannableMap(ruleContext); + + features.addAll(ruleContext.getFeatures()); + } + + private static ImmutableMap<Artifact, IncludeScannable> getLipoScannableMap( + RuleContext ruleContext) { + if (!ruleContext.getFragment(CppConfiguration.class).isLipoOptimization()) { + return null; + } + + LipoContextProvider provider = ruleContext.getPrerequisite( + ":lipo_context_collector", Mode.DONT_CHECK, LipoContextProvider.class); + return provider.getIncludeScannables(); + } + + /** + * Creates a builder for an owner that is not required to be rule. + */ + public CppCompileActionBuilder( + ActionOwner owner, AnalysisEnvironment analysisEnvironment, Artifact sourceFile, + Label sourceLabel, BuildConfiguration configuration) { + this.owner = owner; + this.actionContext = CppCompileActionContext.class; + this.cppConfiguration = configuration.getFragment(CppConfiguration.class); + this.analysisEnvironment = analysisEnvironment; + this.sourceFile = sourceFile; + this.sourceLabel = sourceLabel; + this.configuration = configuration; + this.mandatoryInputsBuilder = NestedSetBuilder.stableOrder(); + this.pluginInputsBuilder = NestedSetBuilder.stableOrder(); + this.lipoScannableMap = ImmutableMap.of(); + } + + /** + * Creates a builder that is a copy of another builder. + */ + public CppCompileActionBuilder(CppCompileActionBuilder other) { + this.owner = other.owner; + this.features.addAll(other.features); + this.featureConfiguration = other.featureConfiguration; + this.sourceFile = other.sourceFile; + this.sourceLabel = other.sourceLabel; + this.mandatoryInputsBuilder = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(other.mandatoryInputsBuilder.build()); + this.pluginInputsBuilder = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(other.pluginInputsBuilder.build()); + this.optionalSourceFile = other.optionalSourceFile; + this.outputFile = other.outputFile; + this.tempOutputFile = other.tempOutputFile; + this.dotdFile = other.dotdFile; + this.gcnoFile = other.gcnoFile; + this.configuration = other.configuration; + this.context = other.context; + this.copts.addAll(other.copts); + this.pluginOpts.addAll(other.pluginOpts); + this.nocopts.addAll(other.nocopts); + this.analysisEnvironment = other.analysisEnvironment; + this.extraSystemIncludePrefixes = ImmutableList.copyOf(other.extraSystemIncludePrefixes); + this.enableLayeringCheck = other.enableLayeringCheck; + this.compileHeaderModules = other.compileHeaderModules; + this.includeResolver = other.includeResolver; + this.actionClassId = other.actionClassId; + this.actionContext = other.actionContext; + this.cppConfiguration = other.cppConfiguration; + this.lipoScannableMap = other.lipoScannableMap; + } + + public PathFragment getTempOutputFile() { + return tempOutputFile; + } + + public Artifact getSourceFile() { + return sourceFile; + } + + public CppCompilationContext getContext() { + return context; + } + + public NestedSet<Artifact> getMandatoryInputs() { + return mandatoryInputsBuilder.build(); + } + + /** + * Returns the .dwo output file that matches the specified .o output file. If Fission mode + * isn't enabled for this build, this is null (we don't produce .dwo files in that case). + */ + private static Artifact getDwoFile(Artifact outputFile, AnalysisEnvironment artifactFactory, + CppConfiguration cppConfiguration) { + + // Only create .dwo's for .o compilations (i.e. not .ii or .S). + boolean isObjectOutput = CppFileTypes.OBJECT_FILE.matches(outputFile.getExecPath()) + || CppFileTypes.PIC_OBJECT_FILE.matches(outputFile.getExecPath()); + + // Note configurations can be null for tests. + if (cppConfiguration != null && cppConfiguration.useFission() && isObjectOutput) { + return artifactFactory.getDerivedArtifact( + FileSystemUtils.replaceExtension(outputFile.getRootRelativePath(), ".dwo"), + outputFile.getRoot()); + } else { + return null; + } + } + + private static Predicate<String> getNocoptPredicate(Collection<Pattern> patterns) { + final ImmutableList<Pattern> finalPatterns = ImmutableList.copyOf(patterns); + if (finalPatterns.isEmpty()) { + return Predicates.alwaysTrue(); + } else { + return new Predicate<String>() { + @Override + public boolean apply(String option) { + for (Pattern pattern : finalPatterns) { + if (pattern.matcher(option).matches()) { + return false; + } + } + + return true; + } + }; + } + } + + private Iterable<IncludeScannable> getLipoScannables(NestedSet<Artifact> realMandatoryInputs) { + return lipoScannableMap == null ? ImmutableList.<IncludeScannable>of() : Iterables.filter( + Iterables.transform( + Iterables.filter( + FileType.filter( + realMandatoryInputs, + CppFileTypes.C_SOURCE, CppFileTypes.CPP_SOURCE, + CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR), + Predicates.not(Predicates.equalTo(getSourceFile()))), + Functions.forMap(lipoScannableMap, null)), + Predicates.notNull()); + } + + /** + * Builds the Action as configured and returns the to be generated Artifact. + * + * <p>This method may be called multiple times to create multiple compile + * actions (usually after calling some setters to modify the generated + * action). + */ + public CppCompileAction build() { + // Configuration can be null in tests. + NestedSetBuilder<Artifact> realMandatoryInputsBuilder = NestedSetBuilder.compileOrder(); + realMandatoryInputsBuilder.addTransitive(mandatoryInputsBuilder.build()); + if (tempOutputFile == null && configuration != null + && !configuration.getFragment(CppConfiguration.class).shouldScanIncludes()) { + realMandatoryInputsBuilder.addTransitive(context.getDeclaredIncludeSrcs()); + } + realMandatoryInputsBuilder.addTransitive(context.getAdditionalInputs()); + realMandatoryInputsBuilder.addTransitive(pluginInputsBuilder.build()); + realMandatoryInputsBuilder.add(sourceFile); + boolean fake = tempOutputFile != null; + + // Copying the collections is needed to make the builder reusable. + if (fake) { + return new FakeCppCompileAction(owner, ImmutableList.copyOf(features), featureConfiguration, + sourceFile, sourceLabel, realMandatoryInputsBuilder.build(), outputFile, tempOutputFile, + dotdFile, configuration, cppConfiguration, context, ImmutableList.copyOf(copts), + ImmutableList.copyOf(pluginOpts), getNocoptPredicate(nocopts), + extraSystemIncludePrefixes, enableLayeringCheck, fdoBuildStamp); + } else { + NestedSet<Artifact> realMandatoryInputs = realMandatoryInputsBuilder.build(); + + return new CppCompileAction(owner, ImmutableList.copyOf(features), featureConfiguration, + sourceFile, sourceLabel, realMandatoryInputs, outputFile, dotdFile, + gcnoFile, getDwoFile(outputFile, analysisEnvironment, cppConfiguration), + optionalSourceFile, configuration, cppConfiguration, context, + actionContext, ImmutableList.copyOf(copts), + ImmutableList.copyOf(pluginOpts), + getNocoptPredicate(nocopts), + extraSystemIncludePrefixes, enableLayeringCheck, fdoBuildStamp, + includeResolver, getLipoScannables(realMandatoryInputs), actionClassId, + compileHeaderModules); + } + } + + /** + * Sets the feature configuration to be used for the action. + */ + public CppCompileActionBuilder setFeatureConfiguration( + FeatureConfiguration featureConfiguration) { + this.featureConfiguration = featureConfiguration; + return this; + } + + public CppCompileActionBuilder setIncludeResolver(IncludeResolver includeResolver) { + this.includeResolver = includeResolver; + return this; + } + + public CppCompileActionBuilder setCppConfiguration(CppConfiguration cppConfiguration) { + this.cppConfiguration = cppConfiguration; + return this; + } + + public CppCompileActionBuilder setActionContext( + Class<? extends CppCompileActionContext> actionContext) { + this.actionContext = actionContext; + return this; + } + + public CppCompileActionBuilder setActionClassId(UUID uuid) { + this.actionClassId = uuid; + return this; + } + + public CppCompileActionBuilder setExtraSystemIncludePrefixes( + Collection<PathFragment> extraSystemIncludePrefixes) { + this.extraSystemIncludePrefixes = ImmutableList.copyOf(extraSystemIncludePrefixes); + return this; + } + + public CppCompileActionBuilder addPluginInput(Artifact artifact) { + pluginInputsBuilder.add(artifact); + return this; + } + + public CppCompileActionBuilder clearPluginInputs() { + pluginInputsBuilder = NestedSetBuilder.stableOrder(); + return this; + } + + /** + * Set an optional source file (usually with metadata of the main source file). The optional + * source file can only be set once, whether via this method or through the constructor + * {@link #CppCompileActionBuilder(CppCompileActionBuilder)}. + */ + public CppCompileActionBuilder addOptionalSourceFile(Artifact artifact) { + Preconditions.checkState(optionalSourceFile == null, "%s %s", optionalSourceFile, artifact); + optionalSourceFile = artifact; + return this; + } + + public CppCompileActionBuilder addMandatoryInputs(Iterable<Artifact> artifacts) { + mandatoryInputsBuilder.addAll(artifacts); + return this; + } + + public CppCompileActionBuilder addTransitiveMandatoryInputs(NestedSet<Artifact> artifacts) { + mandatoryInputsBuilder.addTransitive(artifacts); + return this; + } + + public CppCompileActionBuilder setOutputFile(Artifact outputFile) { + this.outputFile = outputFile; + return this; + } + + /** + * The temp output file is not an artifact, since it does not appear in the outputs of the + * action. + * + * <p>This is theoretically a problem if that file already existed before, since then Blaze + * does not delete it before executing the rule, but 1. that only applies for local + * execution which does not happen very often and 2. it is only a problem if the compiler is + * affected by the presence of this file, which it should not be. + */ + public CppCompileActionBuilder setTempOutputFile(PathFragment tempOutputFile) { + this.tempOutputFile = tempOutputFile; + return this; + } + + @VisibleForTesting + public CppCompileActionBuilder setDotdFileForTesting(Artifact dotdFile) { + this.dotdFile = new DotdFile(dotdFile); + return this; + } + + public CppCompileActionBuilder setDotdFile(PathFragment outputName, String extension, + RuleContext ruleContext) { + if (configuration.getFragment(CppConfiguration.class).getInmemoryDotdFiles()) { + // Just set the path, no artifact is constructed + PathFragment file = FileSystemUtils.replaceExtension(outputName, extension); + Root root = configuration.getBinDirectory(); + dotdFile = new DotdFile(root.getExecPath().getRelative(file)); + } else { + dotdFile = new DotdFile(ruleContext.getRelatedArtifact(outputName, extension)); + } + return this; + } + + public CppCompileActionBuilder setGcnoFile(Artifact gcnoFile) { + this.gcnoFile = gcnoFile; + return this; + } + + public CppCompileActionBuilder addCopt(String copt) { + copts.add(copt); + return this; + } + + public CppCompileActionBuilder addPluginOpt(String opt) { + pluginOpts.add(opt); + return this; + } + + public CppCompileActionBuilder clearPluginOpts() { + pluginOpts.clear(); + return this; + } + + public CppCompileActionBuilder addCopts(Iterable<? extends String> copts) { + Iterables.addAll(this.copts, copts); + return this; + } + + public CppCompileActionBuilder addCopts(int position, Iterable<? extends String> copts) { + this.copts.addAll(position, ImmutableList.copyOf(copts)); + return this; + } + + public CppCompileActionBuilder addNocopts(Pattern nocopts) { + this.nocopts.add(nocopts); + return this; + } + + public CppCompileActionBuilder setContext(CppCompilationContext context) { + this.context = context; + return this; + } + + public CppCompileActionBuilder setEnableLayeringCheck(boolean enableLayeringCheck) { + this.enableLayeringCheck = enableLayeringCheck; + return this; + } + + /** + * Sets whether the CompileAction should use header modules for its compilation. + */ + public CppCompileActionBuilder setCompileHeaderModules(boolean compileHeaderModules) { + this.compileHeaderModules = compileHeaderModules; + return this; + } + + public CppCompileActionBuilder setFdoBuildStamp(String fdoBuildStamp) { + this.fdoBuildStamp = fdoBuildStamp; + return this; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionContext.java new file mode 100644 index 0000000000..4bbfa44d61 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionContext.java @@ -0,0 +1,84 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.ActionContextMarker; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.ResourceSet; + +import java.io.IOException; +import java.util.Collection; + +import javax.annotation.Nullable; + +/** + * Context for compiling plain C++. + */ +@ActionContextMarker(name = "C++") +public interface CppCompileActionContext extends ActionContext { + /** + * Reply for the execution of a C++ compilation. + */ + public interface Reply { + /** + * Returns the contents of the .d file. + */ + byte[] getContents() throws IOException; + } + + /** Does include scanning to find the list of files needed to execute the action. */ + public Collection<? extends ActionInput> findAdditionalInputs(CppCompileAction action, + ActionExecutionContext actionExecutionContext) + throws ExecException, InterruptedException, ActionExecutionException; + + /** + * Executes the given action and return the reply of the executor. + */ + Reply execWithReply(CppCompileAction action, + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException; + + /** + * Returns the executor reply from an exec exception, if available. + */ + @Nullable Reply getReplyFromException( + ExecException e, CppCompileAction action); + + /** + * Returns the estimated resource consumption of the action. + */ + ResourceSet estimateResourceConsumption(CppCompileAction action); + + /** + * Returns where the action actually runs. + */ + String strategyLocality(); + + /** + * Returns whether include scanning needs to be run. + */ + boolean needsIncludeScanning(); + + /** + * Returns the include files that should be shipped to the executor in addition the ones that + * were declared. + */ + Collection<Artifact> getScannedIncludeFiles( + CppCompileAction action, ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java new file mode 100644 index 0000000000..c5cc9a57fc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java @@ -0,0 +1,1691 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import com.google.common.collect.Iterables; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.PackageRootResolver; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.ViewCreationFailedException; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.CompilationMode; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.analysis.config.PerLabelOptions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.rules.cpp.CppConfigurationLoader.CppConfigurationParameters; +import com.google.devtools.build.lib.rules.cpp.FdoSupport.FdoException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.util.IncludeScanningUtil; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LinkingModeFlags; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode; +import com.google.devtools.build.skyframe.SkyFunction.Environment; +import com.google.devtools.common.options.OptionsParsingException; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipException; + +/** + * This class represents the C/C++ parts of the {@link BuildConfiguration}, + * including the host architecture, target architecture, compiler version, and + * a standard library version. It has information about the tools locations and + * the flags required for compiling. + */ +@SkylarkModule(name = "cpp", doc = "A configuration fragment for C++") +@Immutable +public class CppConfiguration extends BuildConfiguration.Fragment { + /** + * An enumeration of all the tools that comprise a toolchain. + */ + public enum Tool { + AR("ar"), + CPP("cpp"), + GCC("gcc"), + GCOV("gcov"), + GCOVTOOL("gcov-tool"), + LD("ld"), + NM("nm"), + OBJCOPY("objcopy"), + OBJDUMP("objdump"), + STRIP("strip"), + DWP("dwp"); + + private final String namePart; + + private Tool(String namePart) { + this.namePart = namePart; + } + + public String getNamePart() { + return namePart; + } + } + + /** + * Values for the --hdrs_check option. + */ + public static enum HeadersCheckingMode { + /** Legacy behavior: Silently allow undeclared headers. */ + LOOSE, + /** Warn about undeclared headers. */ + WARN, + /** Disallow undeclared headers. */ + STRICT + } + + /** + * --dynamic_mode parses to DynamicModeFlag, but AUTO will be translated based on platform, + * resulting in a DynamicMode value. + */ + public enum DynamicMode { OFF, DEFAULT, FULLY } + + /** + * This enumeration is used for the --strip option. + */ + public static enum StripMode { + + ALWAYS("always"), // Always strip. + SOMETIMES("sometimes"), // Strip iff compilationMode == FASTBUILD. + NEVER("never"); // Never strip. + + private final String mode; + + private StripMode(String mode) { + this.mode = mode; + } + + @Override + public String toString() { + return mode; + } + } + + /** Storage for the libc label, if given. */ + public static class LibcTop implements Serializable { + private final Label label; + + LibcTop(Label label) { + Preconditions.checkArgument(label != null); + this.label = label; + } + + public Label getLabel() { + return label; + } + + public PathFragment getSysroot() { + return label.getPackageFragment(); + } + + @Override + public String toString() { + return label.toString(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } else if (other instanceof LibcTop) { + return label.equals(((LibcTop) other).label); + } else { + return false; + } + } + + @Override + public int hashCode() { + return label.hashCode(); + } + } + + /** + * This macro will be passed as a command-line parameter (eg. -DBUILD_FDO_TYPE="LIPO"). + * For possible values see {@code CppModel.getFdoBuildStamp()}. + */ + public static final String FDO_STAMP_MACRO = "BUILD_FDO_TYPE"; + + /** + * Represents an optional flag that can be toggled using the package features mechanism. + */ + @VisibleForTesting + static class OptionalFlag implements Serializable { + private final String name; + private final List<String> flags; + + @VisibleForTesting + OptionalFlag(String name, List<String> flags) { + this.name = name; + this.flags = flags; + } + + private List<String> getFlags() { + return flags; + } + + private String getName() { + return name; + } + } + + @VisibleForTesting + static class FlagList implements Serializable { + private List<String> prefixFlags; + private List<OptionalFlag> optionalFlags; + private List<String> suffixFlags; + + @VisibleForTesting + FlagList(List<String> prefixFlags, + List<OptionalFlag> optionalFlags, + List<String> suffixFlags) { + this.prefixFlags = prefixFlags; + this.optionalFlags = optionalFlags; + this.suffixFlags = suffixFlags; + } + + @VisibleForTesting + List<String> evaluate(Collection<String> features) { + ImmutableList.Builder<String> result = ImmutableList.builder(); + result.addAll(prefixFlags); + for (OptionalFlag optionalFlag : optionalFlags) { + // The flag is added if the default is true and the flag is not specified, + // or if the default is false and the flag is specified. + if (features.contains(optionalFlag.getName())) { + result.addAll(optionalFlag.getFlags()); + } + } + + result.addAll(suffixFlags); + return result.build(); + } + } + + private final Label crosstoolTop; + private final String hostSystemName; + private final String compiler; + private final String targetCpu; + private final String targetSystemName; + private final String targetLibc; + private final LipoMode lipoMode; + private final PathFragment crosstoolTopPathFragment; + + private final String abi; + private final String abiGlibcVersion; + + private final String toolchainIdentifier; + private final String cacheKey; + + private final CcToolchainFeatures toolchainFeatures; + private final boolean supportsGoldLinker; + private final boolean supportsThinArchives; + private final boolean supportsStartEndLib; + private final boolean supportsInterfaceSharedObjects; + private final boolean supportsEmbeddedRuntimes; + private final boolean supportsFission; + + // We encode three states with two booleans: + // (1) (false false) -> no pic code + // (2) (true false) -> shared libraries as pic, but not binaries + // (3) (true true) -> both shared libraries and binaries as pic + private final boolean toolchainNeedsPic; + private final boolean usePicForBinaries; + + private final FdoSupport fdoSupport; + + // TODO(bazel-team): All these labels (except for ccCompilerRuleLabel) can be removed once the + // transition to the cc_compiler rule is complete. + private final Label libcLabel; + private final Label staticRuntimeLibsLabel; + private final Label dynamicRuntimeLibsLabel; + private final Label ccToolchainLabel; + + private final PathFragment sysroot; + private final PathFragment runtimeSysroot; + private final List<PathFragment> builtInIncludeDirectories; + + private final Map<String, PathFragment> toolPaths; + private final PathFragment ldExecutable; + + // Only used during construction. + private final List<String> commonLinkOptions; + private final ListMultimap<CompilationMode, String> linkOptionsFromCompilationMode; + private final ListMultimap<LipoMode, String> linkOptionsFromLipoMode; + private final ListMultimap<LinkingMode, String> linkOptionsFromLinkingMode; + + private final FlagList compilerFlags; + private final FlagList cxxFlags; + private final FlagList unfilteredCompilerFlags; + private final List<String> cOptions; + + private FlagList fullyStaticLinkFlags; + private FlagList mostlyStaticLinkFlags; + private FlagList mostlyStaticSharedLinkFlags; + private FlagList dynamicLinkFlags; + private FlagList dynamicLibraryLinkFlags; + private final List<String> testOnlyLinkFlags; + + private final List<String> linkOptions; + + private final List<String> objcopyOptions; + private final List<String> ldOptions; + private final List<String> arOptions; + private final List<String> arThinArchivesOptions; + + private final Map<String, String> additionalMakeVariables; + + private final CppOptions cppOptions; + + // The dynamic mode for linking. + private final DynamicMode dynamicMode; + private final boolean stripBinaries; + private final ImmutableMap<String, String> commandLineDefines; + private final String solibDirectory; + private final CompilationMode compilationMode; + private final Path execRoot; + /** + * If true, the ConfiguredTarget is only used to get the necessary cross-referenced + * CppCompilationContexts, but registering build actions is disabled. + */ + private final boolean lipoContextCollector; + private final Root greppedIncludesDirectory; + + protected CppConfiguration(CppConfigurationParameters params) + throws InvalidConfigurationException { + CrosstoolConfig.CToolchain toolchain = params.toolchain; + cppOptions = params.buildOptions.get(CppOptions.class); + this.hostSystemName = toolchain.getHostSystemName(); + this.compiler = toolchain.getCompiler(); + this.targetCpu = toolchain.getTargetCpu(); + this.lipoMode = cppOptions.getLipoMode(); + this.targetSystemName = toolchain.getTargetSystemName(); + this.targetLibc = toolchain.getTargetLibc(); + this.crosstoolTop = params.crosstoolTop; + this.ccToolchainLabel = params.ccToolchainLabel; + this.compilationMode = + params.buildOptions.get(BuildConfiguration.Options.class).compilationMode; + this.lipoContextCollector = cppOptions.lipoCollector; + this.execRoot = params.execRoot; + + // Note that the grepped includes directory is not configuration-specific; the paths of the + // files within that directory, however, are configuration-specific. + this.greppedIncludesDirectory = Root.asDerivedRoot(execRoot, + execRoot.getRelative(IncludeScanningUtil.GREPPED_INCLUDES)); + + this.crosstoolTopPathFragment = crosstoolTop.getPackageFragment(); + + try { + this.staticRuntimeLibsLabel = + crosstoolTop.getRelative(toolchain.hasStaticRuntimesFilegroup() ? + toolchain.getStaticRuntimesFilegroup() : "static-runtime-libs-" + targetCpu); + this.dynamicRuntimeLibsLabel = + crosstoolTop.getRelative(toolchain.hasDynamicRuntimesFilegroup() ? + toolchain.getDynamicRuntimesFilegroup() : "dynamic-runtime-libs-" + targetCpu); + } catch (SyntaxException e) { + // All of the above label.getRelative() calls are valid labels, and the crosstool_top + // was already checked earlier in the process. + throw new AssertionError(e); + } + + if (cppOptions.lipoMode == LipoMode.BINARY) { + // TODO(bazel-team): implement dynamic linking with LIPO + this.dynamicMode = DynamicMode.OFF; + } else { + switch (cppOptions.dynamicMode) { + case DEFAULT: + this.dynamicMode = DynamicMode.DEFAULT; break; + case OFF: this.dynamicMode = DynamicMode.OFF; break; + case FULLY: this.dynamicMode = DynamicMode.FULLY; break; + default: throw new IllegalStateException("Invalid dynamicMode."); + } + } + + this.fdoSupport = new FdoSupport( + params.buildOptions.get(CppOptions.class).fdoInstrument, params.fdoZip, + cppOptions.lipoMode, execRoot); + + this.stripBinaries = (cppOptions.stripBinaries == StripMode.ALWAYS || + (cppOptions.stripBinaries == StripMode.SOMETIMES && + compilationMode == CompilationMode.FASTBUILD)); + + CrosstoolConfigurationIdentifier crosstoolConfig = + CrosstoolConfigurationIdentifier.fromToolchain(toolchain); + Preconditions.checkState(crosstoolConfig.getCpu().equals(targetCpu)); + Preconditions.checkState(crosstoolConfig.getCompiler().equals(compiler)); + Preconditions.checkState(crosstoolConfig.getLibc().equals(targetLibc)); + + this.solibDirectory = "_solib_" + targetCpu; + + this.toolchainIdentifier = toolchain.getToolchainIdentifier(); + this.cacheKey = this + ":" + crosstoolTop + ":" + params.cacheKeySuffix + ":" + + lipoContextCollector; + + this.toolchainFeatures = new CcToolchainFeatures(toolchain); + this.supportsGoldLinker = toolchain.getSupportsGoldLinker(); + this.supportsThinArchives = toolchain.getSupportsThinArchives(); + this.supportsStartEndLib = toolchain.getSupportsStartEndLib(); + this.supportsInterfaceSharedObjects = toolchain.getSupportsInterfaceSharedObjects(); + this.supportsEmbeddedRuntimes = toolchain.getSupportsEmbeddedRuntimes(); + this.supportsFission = toolchain.getSupportsFission(); + this.toolchainNeedsPic = toolchain.getNeedsPic(); + this.usePicForBinaries = + toolchain.getNeedsPic() && compilationMode != CompilationMode.OPT; + + this.toolPaths = Maps.newHashMap(); + for (CrosstoolConfig.ToolPath tool : toolchain.getToolPathList()) { + PathFragment path = new PathFragment(tool.getPath()); + if (!path.isNormalized()) { + throw new IllegalArgumentException("The include path '" + tool.getPath() + + "' is not normalized."); + } + toolPaths.put(tool.getName(), crosstoolTopPathFragment.getRelative(path)); + } + + if (toolPaths.isEmpty()) { + // If no paths are specified, we just use the names of the tools as the path. + for (Tool tool : Tool.values()) { + toolPaths.put(tool.getNamePart(), + crosstoolTopPathFragment.getRelative(tool.getNamePart())); + } + } else { + Iterable<Tool> neededTools = Iterables.filter(EnumSet.allOf(Tool.class), + new Predicate<Tool>() { + @Override + public boolean apply(Tool tool) { + if (tool == Tool.DWP) { + // When fission is unsupported, don't check for the dwp tool. + return supportsFission(); + } else if (tool == Tool.GCOVTOOL) { + // gcov-tool is optional, don't check whether it's present + return false; + } else { + return true; + } + } + }); + for (Tool tool : neededTools) { + if (!toolPaths.containsKey(tool.getNamePart())) { + throw new IllegalArgumentException("Tool path for '" + tool.getNamePart() + + "' is missing"); + } + } + } + + // We can't use an ImmutableMap.Builder here; we need the ability (at least + // in tests) to add entries with keys that are already in the map, and only + // HashMap supports this (by replacing the existing entry under the key). + Map<String, String> commandLineDefinesBuilder = new HashMap<>(); + for (Map.Entry<String, String> define : cppOptions.commandLineDefinedVariables) { + commandLineDefinesBuilder.put(define.getKey(), define.getValue()); + } + commandLineDefines = ImmutableMap.copyOf(commandLineDefinesBuilder); + + ListMultimap<CompilationMode, String> cFlags = ArrayListMultimap.create(); + ListMultimap<CompilationMode, String> cxxFlags = ArrayListMultimap.create(); + linkOptionsFromCompilationMode = ArrayListMultimap.create(); + for (CrosstoolConfig.CompilationModeFlags flags : toolchain.getCompilationModeFlagsList()) { + // Remove this when CROSSTOOL files no longer contain 'coverage'. + if (flags.getMode() == CrosstoolConfig.CompilationMode.COVERAGE) { + continue; + } + CompilationMode realmode = importCompilationMode(flags.getMode()); + cFlags.putAll(realmode, flags.getCompilerFlagList()); + cxxFlags.putAll(realmode, flags.getCxxFlagList()); + linkOptionsFromCompilationMode.putAll(realmode, flags.getLinkerFlagList()); + } + + ListMultimap<LipoMode, String> lipoCFlags = ArrayListMultimap.create(); + ListMultimap<LipoMode, String> lipoCxxFlags = ArrayListMultimap.create(); + linkOptionsFromLipoMode = ArrayListMultimap.create(); + for (CrosstoolConfig.LipoModeFlags flags : toolchain.getLipoModeFlagsList()) { + LipoMode realmode = flags.getMode(); + lipoCFlags.putAll(realmode, flags.getCompilerFlagList()); + lipoCxxFlags.putAll(realmode, flags.getCxxFlagList()); + linkOptionsFromLipoMode.putAll(realmode, flags.getLinkerFlagList()); + } + + linkOptionsFromLinkingMode = ArrayListMultimap.create(); + for (LinkingModeFlags flags : toolchain.getLinkingModeFlagsList()) { + LinkingMode realmode = importLinkingMode(flags.getMode()); + linkOptionsFromLinkingMode.putAll(realmode, flags.getLinkerFlagList()); + } + + this.commonLinkOptions = ImmutableList.copyOf(toolchain.getLinkerFlagList()); + dynamicLibraryLinkFlags = new FlagList( + ImmutableList.copyOf(toolchain.getDynamicLibraryLinkerFlagList()), + convertOptionalOptions(toolchain.getOptionalDynamicLibraryLinkerFlagList()), + Collections.<String>emptyList()); + this.objcopyOptions = ImmutableList.copyOf(toolchain.getObjcopyEmbedFlagList()); + this.ldOptions = ImmutableList.copyOf(toolchain.getLdEmbedFlagList()); + this.arOptions = copyOrDefaultIfEmpty(toolchain.getArFlagList(), "rcsD"); + this.arThinArchivesOptions = copyOrDefaultIfEmpty( + toolchain.getArThinArchivesFlagList(), "rcsDT"); + + this.abi = toolchain.getAbiVersion(); + this.abiGlibcVersion = toolchain.getAbiLibcVersion(); + + // The default value for optional string attributes is the empty string. + PathFragment defaultSysroot = toolchain.getBuiltinSysroot().length() == 0 + ? null + : new PathFragment(toolchain.getBuiltinSysroot()); + if ((defaultSysroot != null) && !defaultSysroot.isNormalized()) { + throw new IllegalArgumentException("The built-in sysroot '" + defaultSysroot + + "' is not normalized."); + } + + if ((cppOptions.libcTop != null) && (defaultSysroot == null)) { + throw new InvalidConfigurationException("The selected toolchain " + toolchainIdentifier + + " does not support setting --grte_top."); + } + LibcTop libcTop = cppOptions.libcTop; + if ((libcTop == null) && !toolchain.getDefaultGrteTop().isEmpty()) { + try { + libcTop = new CppOptions.LibcTopConverter().convert(toolchain.getDefaultGrteTop()); + } catch (OptionsParsingException e) { + throw new InvalidConfigurationException(e.getMessage(), e); + } + } + if ((libcTop != null) && (libcTop.getLabel() != null)) { + libcLabel = libcTop.getLabel(); + } else { + libcLabel = null; + } + + ImmutableList.Builder<PathFragment> builtInIncludeDirectoriesBuilder + = ImmutableList.builder(); + sysroot = libcTop == null ? defaultSysroot : libcTop.getSysroot(); + for (String s : toolchain.getCxxBuiltinIncludeDirectoryList()) { + builtInIncludeDirectoriesBuilder.add( + resolveIncludeDir(s, sysroot, crosstoolTopPathFragment)); + } + builtInIncludeDirectories = builtInIncludeDirectoriesBuilder.build(); + + // The runtime sysroot should really be set from --grte_top. However, currently libc has no + // way to set the sysroot. The CROSSTOOL file does set the runtime sysroot, in the + // builtin_sysroot field. This implies that you can not arbitrarily mix and match Crosstool + // and libc versions, you must always choose compatible ones. + runtimeSysroot = defaultSysroot; + + String sysrootFlag; + if (sysroot != null && !sysroot.equals(defaultSysroot)) { + // Only specify the --sysroot option if it is different from the built-in one. + sysrootFlag = "--sysroot=" + sysroot; + } else { + sysrootFlag = null; + } + + ImmutableList.Builder<String> unfilteredCoptsBuilder = ImmutableList.builder(); + if (sysrootFlag != null) { + unfilteredCoptsBuilder.add(sysrootFlag); + } + unfilteredCoptsBuilder.addAll(toolchain.getUnfilteredCxxFlagList()); + unfilteredCompilerFlags = new FlagList( + unfilteredCoptsBuilder.build(), + convertOptionalOptions(toolchain.getOptionalUnfilteredCxxFlagList()), + Collections.<String>emptyList()); + + ImmutableList.Builder<String> linkoptsBuilder = ImmutableList.builder(); + linkoptsBuilder.addAll(cppOptions.linkoptList); + if (cppOptions.experimentalOmitfp) { + linkoptsBuilder.add("-Wl,--eh-frame-hdr"); + } + if (sysrootFlag != null) { + linkoptsBuilder.add(sysrootFlag); + } + this.linkOptions = linkoptsBuilder.build(); + + ImmutableList.Builder<String> coptsBuilder = ImmutableList.<String>builder() + .addAll(toolchain.getCompilerFlagList()) + .addAll(cFlags.get(compilationMode)) + .addAll(lipoCFlags.get(cppOptions.getLipoMode())); + if (cppOptions.experimentalOmitfp) { + coptsBuilder.add("-fomit-frame-pointer"); + coptsBuilder.add("-fasynchronous-unwind-tables"); + coptsBuilder.add("-DNO_FRAME_POINTER"); + } + this.compilerFlags = new FlagList( + coptsBuilder.build(), + convertOptionalOptions(toolchain.getOptionalCompilerFlagList()), + cppOptions.coptList); + + this.cOptions = ImmutableList.copyOf(cppOptions.conlyoptList); + + ImmutableList.Builder<String> cxxOptsBuilder = ImmutableList.<String>builder() + .addAll(toolchain.getCxxFlagList()) + .addAll(cxxFlags.get(compilationMode)) + .addAll(lipoCxxFlags.get(cppOptions.getLipoMode())); + + this.cxxFlags = new FlagList( + cxxOptsBuilder.build(), + convertOptionalOptions(toolchain.getOptionalCxxFlagList()), + cppOptions.cxxoptList); + + this.ldExecutable = getToolPathFragment(CppConfiguration.Tool.LD); + + boolean stripBinaries = (cppOptions.stripBinaries == StripMode.ALWAYS) || + ((cppOptions.stripBinaries == StripMode.SOMETIMES) && + (compilationMode == CompilationMode.FASTBUILD)); + + fullyStaticLinkFlags = new FlagList( + configureLinkerOptions(compilationMode, lipoMode, LinkingMode.FULLY_STATIC, + ldExecutable, stripBinaries), + convertOptionalOptions(toolchain.getOptionalLinkerFlagList()), + Collections.<String>emptyList()); + mostlyStaticLinkFlags = new FlagList( + configureLinkerOptions(compilationMode, lipoMode, LinkingMode.MOSTLY_STATIC, + ldExecutable, stripBinaries), + convertOptionalOptions(toolchain.getOptionalLinkerFlagList()), + Collections.<String>emptyList()); + mostlyStaticSharedLinkFlags = new FlagList( + configureLinkerOptions(compilationMode, lipoMode, + LinkingMode.MOSTLY_STATIC_LIBRARIES, ldExecutable, stripBinaries), + convertOptionalOptions(toolchain.getOptionalLinkerFlagList()), + Collections.<String>emptyList()); + dynamicLinkFlags = new FlagList( + configureLinkerOptions(compilationMode, lipoMode, LinkingMode.DYNAMIC, + ldExecutable, stripBinaries), + convertOptionalOptions(toolchain.getOptionalLinkerFlagList()), + Collections.<String>emptyList()); + testOnlyLinkFlags = ImmutableList.copyOf(toolchain.getTestOnlyLinkerFlagList()); + + Map<String, String> makeVariablesBuilder = new HashMap<>(); + // The following are to be used to allow some build rules to avoid the limits on stack frame + // sizes and variable-length arrays. Ensure that these are always set. + makeVariablesBuilder.put("STACK_FRAME_UNLIMITED", ""); + makeVariablesBuilder.put("CC_FLAGS", ""); + for (CrosstoolConfig.MakeVariable variable : toolchain.getMakeVariableList()) { + makeVariablesBuilder.put(variable.getName(), variable.getValue()); + } + if (sysrootFlag != null) { + String ccFlags = makeVariablesBuilder.get("CC_FLAGS"); + ccFlags = ccFlags.isEmpty() ? sysrootFlag : ccFlags + " " + sysrootFlag; + makeVariablesBuilder.put("CC_FLAGS", ccFlags); + } + this.additionalMakeVariables = ImmutableMap.copyOf(makeVariablesBuilder); + } + + private List<OptionalFlag> convertOptionalOptions( + List<CrosstoolConfig.CToolchain.OptionalFlag> optionalFlagList) + throws IllegalArgumentException { + List<OptionalFlag> result = new ArrayList<>(); + + for (CrosstoolConfig.CToolchain.OptionalFlag crosstoolOptionalFlag : optionalFlagList) { + String name = crosstoolOptionalFlag.getDefaultSettingName(); + result.add(new OptionalFlag( + name, + ImmutableList.copyOf(crosstoolOptionalFlag.getFlagList()))); + } + + return result; + } + + private static ImmutableList<String> copyOrDefaultIfEmpty(List<String> list, + String defaultValue) { + return list.isEmpty() ? ImmutableList.of(defaultValue) : ImmutableList.copyOf(list); + } + + @VisibleForTesting + static CompilationMode importCompilationMode(CrosstoolConfig.CompilationMode mode) { + return CompilationMode.valueOf(mode.name()); + } + + @VisibleForTesting + static LinkingMode importLinkingMode(CrosstoolConfig.LinkingMode mode) { + return LinkingMode.valueOf(mode.name()); + } + + private static final PathFragment SYSROOT_FRAGMENT = new PathFragment("%sysroot%"); + + /** + * Resolve the given include directory. If it is not absolute, it is + * interpreted relative to the crosstool top. If it starts with %sysroot%/, + * that part is replaced with the actual sysroot. + */ + static PathFragment resolveIncludeDir(String s, PathFragment sysroot, + PathFragment crosstoolTopPathFragment) { + PathFragment path = new PathFragment(s); + if (!path.isNormalized()) { + throw new IllegalArgumentException("The include path '" + s + "' is not normalized."); + } + if (path.startsWith(SYSROOT_FRAGMENT)) { + if (sysroot == null) { + throw new IllegalArgumentException("A %sysroot% prefix is only allowed if the " + + "default_sysroot option is set"); + } + return sysroot.getRelative(path.relativeTo(SYSROOT_FRAGMENT)); + } else { + return crosstoolTopPathFragment.getRelative(path); + } + } + + /** + * Returns the configuration-independent grepped-includes directory. + */ + public Root getGreppedIncludesDirectory() { + return greppedIncludesDirectory; + } + + @VisibleForTesting + List<String> configureLinkerOptions( + CompilationMode compilationMode, LipoMode lipoMode, LinkingMode linkingMode, + PathFragment ldExecutable, boolean stripBinaries) { + List<String> result = new ArrayList<>(); + result.addAll(commonLinkOptions); + + result.add("-B" + ldExecutable.getParentDirectory().getPathString()); + if (stripBinaries) { + result.add("-Wl,-S"); + } + + result.addAll(linkOptionsFromCompilationMode.get(compilationMode)); + result.addAll(linkOptionsFromLipoMode.get(lipoMode)); + result.addAll(linkOptionsFromLinkingMode.get(linkingMode)); + return ImmutableList.copyOf(result); + } + + /** + * Returns the toolchain identifier, which uniquely identifies the compiler + * version, target libc version, target cpu, and LIPO linkage. + */ + public String getToolchainIdentifier() { + return toolchainIdentifier; + } + + /** + * Returns the system name which is required by the toolchain to run. + */ + public String getHostSystemName() { + return hostSystemName; + } + + @Override + public String toString() { + return toolchainIdentifier; + } + + /** + * Returns the compiler version string (e.g. "gcc-4.1.1"). + */ + @SkylarkCallable(name = "compiler", structField = true, doc = "C++ compiler.") + public String getCompiler() { + return compiler; + } + + /** + * Returns the libc version string (e.g. "glibc-2.2.2"). + */ + public String getTargetLibc() { + return targetLibc; + } + + /** + * Returns the target architecture using blaze-specific constants (e.g. "piii"). + */ + @SkylarkCallable(name = "cpu", structField = true, doc = "Target CPU of the C++ toolchain.") + public String getTargetCpu() { + return targetCpu; + } + + /** + * Returns the path fragment that is either absolute or relative to the + * execution root that can be used to execute the given tool. + * + * <p>Note that you must not use this method to get the linker location, but + * use {@link #getLdExecutable} instead! + */ + public PathFragment getToolPathFragment(CppConfiguration.Tool tool) { + return toolPaths.get(tool.getNamePart()); + } + + /** + * Returns a label that forms a dependency to the files required for the + * sysroot that is used. + */ + public Label getLibcLabel() { + return libcLabel; + } + + /** + * Returns a label that references the library files needed to statically + * link the C++ runtime (i.e. libgcc.a, libgcc_eh.a, libstdc++.a) for the + * target architecture. + */ + public Label getStaticRuntimeLibsLabel() { + return supportsEmbeddedRuntimes() ? staticRuntimeLibsLabel : null; + } + + /** + * Returns a label that references the library files needed to dynamically + * link the C++ runtime (i.e. libgcc_s.so, libstdc++.so) for the target + * architecture. + */ + public Label getDynamicRuntimeLibsLabel() { + return supportsEmbeddedRuntimes() ? dynamicRuntimeLibsLabel : null; + } + + /** + * Returns the label of the <code>cc_compiler</code> rule for the C++ configuration. + */ + public Label getCcToolchainRuleLabel() { + return ccToolchainLabel; + } + + /** + * Returns the abi we're using, which is a gcc version. E.g.: "gcc-3.4". + * Note that in practice we might be using gcc-3.4 as ABI even when compiling + * with gcc-4.1.0, because ABIs are backwards compatible. + */ + // TODO(bazel-team): The javadoc should clarify how this is used in Blaze. + public String getAbi() { + return abi; + } + + /** + * Returns the glibc version used by the abi we're using. This is a + * glibc version number (e.g., "2.2.2"). Note that in practice we + * might be using glibc 2.2.2 as ABI even when compiling with + * gcc-4.2.2, gcc-4.3.1, or gcc-4.4.0 (which use glibc 2.3.6), + * because ABIs are backwards compatible. + */ + // TODO(bazel-team): The javadoc should clarify how this is used in Blaze. + public String getAbiGlibcVersion() { + return abiGlibcVersion; + } + + /** + * Returns the configured features of the toolchain. Rules should not call this directly, but + * instead use {@code CcToolchainProvider.getFeatures}. + */ + public CcToolchainFeatures getFeatures() { + return toolchainFeatures; + } + + /** + * Returns whether the toolchain supports the gold linker. + */ + public boolean supportsGoldLinker() { + return supportsGoldLinker; + } + + /** + * Returns whether the toolchain supports thin archives. + */ + public boolean supportsThinArchives() { + return supportsThinArchives; + } + + /** + * Returns whether the toolchain supports the --start-lib/--end-lib options. + */ + public boolean supportsStartEndLib() { + return supportsStartEndLib; + } + + /** + * Returns whether build_interface_so can build interface shared objects for this toolchain. + * Should be true if this toolchain generates ELF objects. + */ + public boolean supportsInterfaceSharedObjects() { + return supportsInterfaceSharedObjects; + } + + /** + * Returns whether the toolchain supports linking C/C++ runtime libraries + * supplied inside the toolchain distribution. + */ + public boolean supportsEmbeddedRuntimes() { + return supportsEmbeddedRuntimes; + } + + /** + * Returns whether the toolchain supports EXEC_ORIGIN libraries resolution. + */ + public boolean supportsExecOrigin() { + // We're rolling out support for this in the same release that also supports embedded runtimes. + return supportsEmbeddedRuntimes; + } + + /** + * Returns whether the toolchain supports "Fission" C++ builds, i.e. builds + * where compilation partitions object code and debug symbols into separate + * output files. + */ + public boolean supportsFission() { + return supportsFission; + } + + /** + * Returns whether shared libraries must be compiled with position + * independent code on this platform. + */ + public boolean toolchainNeedsPic() { + return toolchainNeedsPic; + } + + /** + * Returns whether binaries must be compiled with position independent code. + */ + public boolean usePicForBinaries() { + return usePicForBinaries; + } + + /** + * Returns the type of archives being used. + */ + public Link.ArchiveType archiveType() { + if (useStartEndLib()) { + return Link.ArchiveType.START_END_LIB; + } + if (useThinArchives()) { + return Link.ArchiveType.THIN; + } + return Link.ArchiveType.FAT; + } + + /** + * Returns the ar flags to be used. + */ + public List<String> getArFlags(boolean thinArchives) { + return thinArchives ? arThinArchivesOptions : arOptions; + } + + /** + * Returns a string that uniquely identifies the toolchain. + */ + @Override + public String cacheKey() { + return cacheKey; + } + + /** + * Returns the built-in list of system include paths for the toolchain + * compiler. All paths in this list should be relative to the exec directory. + * They may be absolute if they are also installed on the remote build nodes or + * for local compilation. + */ + public List<PathFragment> getBuiltInIncludeDirectories() { + return builtInIncludeDirectories; + } + + /** + * Returns the sysroot to be used. If the toolchain compiler does not support + * different sysroots, or the sysroot is the same as the default sysroot, then + * this method returns <code>null</code>. + */ + public PathFragment getSysroot() { + return sysroot; + } + + /** + * Returns the run time sysroot, which is where the dynamic linker + * and system libraries are found at runtime. This is usually an absolute path. If the + * toolchain compiler does not support sysroots, then this method returns <code>null</code>. + */ + public PathFragment getRuntimeSysroot() { + return runtimeSysroot; + } + + /** + * Returns the default options to use for compiling C, C++, and assembler. + * This is just the options that should be used for all three languages. + * There may be additional C-specific or C++-specific options that should be used, + * in addition to the ones returned by this method; + */ + public List<String> getCompilerOptions(Collection<String> features) { + return compilerFlags.evaluate(features); + } + + /** + * Returns the list of additional C-specific options to use for compiling + * C. These should be go on the command line after the common options + * returned by {@link #getCompilerOptions}. + */ + public List<String> getCOptions() { + return cOptions; + } + + /** + * Returns the list of additional C++-specific options to use for compiling + * C++. These should be go on the command line after the common options + * returned by {@link #getCompilerOptions}. + */ + public List<String> getCxxOptions(Collection<String> features) { + return cxxFlags.evaluate(features); + } + + /** + * Returns the default list of options which cannot be filtered by BUILD + * rules. These should be appended to the command line after filtering. + */ + public List<String> getUnfilteredCompilerOptions(Collection<String> features) { + return unfilteredCompilerFlags.evaluate(features); + } + + /** + * Returns the set of command-line linker options, including any flags + * inferred from the command-line options. + * + * @see Link + */ + // TODO(bazel-team): Clean up the linker options computation! + public List<String> getLinkOptions() { + return linkOptions; + } + + /** + * Returns the immutable list of linker options for fully statically linked + * outputs. Does not include command-line options passed via --linkopt or + * --linkopts. + * + * @param features default settings affecting this link + * @param sharedLib true if the output is a shared lib, false if it's an executable + */ + public List<String> getFullyStaticLinkOptions(Collection<String> features, + boolean sharedLib) { + if (sharedLib) { + return getSharedLibraryLinkOptions(mostlyStaticLinkFlags, features); + } else { + return fullyStaticLinkFlags.evaluate(features); + } + } + + /** + * Returns the immutable list of linker options for mostly statically linked + * outputs. Does not include command-line options passed via --linkopt or + * --linkopts. + * + * @param features default settings affecting this link + * @param sharedLib true if the output is a shared lib, false if it's an executable + */ + public List<String> getMostlyStaticLinkOptions(Collection<String> features, + boolean sharedLib) { + if (sharedLib) { + return getSharedLibraryLinkOptions( + supportsEmbeddedRuntimes ? mostlyStaticSharedLinkFlags : dynamicLinkFlags, + features); + } else { + return mostlyStaticLinkFlags.evaluate(features); + } + } + + /** + * Returns the immutable list of linker options for artifacts that are not + * fully or mostly statically linked. Does not include command-line options + * passed via --linkopt or --linkopts. + * + * @param features default settings affecting this link + * @param sharedLib true if the output is a shared lib, false if it's an executable + */ + public List<String> getDynamicLinkOptions(Collection<String> features, + boolean sharedLib) { + if (sharedLib) { + return getSharedLibraryLinkOptions(dynamicLinkFlags, features); + } else { + return dynamicLinkFlags.evaluate(features); + } + } + + /** + * Returns link options for the specified flag list, combined with universal options + * for all shared libraries (regardless of link staticness). + */ + private List<String> getSharedLibraryLinkOptions(FlagList flags, + Collection<String> features) { + return ImmutableList.<String>builder() + .addAll(flags.evaluate(features)) + .addAll(dynamicLibraryLinkFlags.evaluate(features)) + .build(); + } + + /** + * Returns test-only link options such that certain test-specific features can be configured + * separately (e.g. lazy binding). + */ + public List<String> getTestOnlyLinkOptions() { + return testOnlyLinkFlags; + } + + + /** + * Returns the list of options to be used with 'objcopy' when converting + * binary files to object files, or {@code null} if this operation is not + * supported. + */ + public List<String> getObjCopyOptionsForEmbedding() { + return objcopyOptions; + } + + /** + * Returns the list of options to be used with 'ld' when converting + * binary files to object files, or {@code null} if this operation is not + * supported. + */ + public List<String> getLdOptionsForEmbedding() { + return ldOptions; + } + + /** + * Returns a map of additional make variables for use by {@link + * BuildConfiguration}. These are to used to allow some build rules to + * avoid the limits on stack frame sizes and variable-length arrays. + * + * <p>The returned map must contain an entry for {@code STACK_FRAME_UNLIMITED}, + * though the entry may be an empty string. + */ + @VisibleForTesting + public Map<String, String> getAdditionalMakeVariables() { + return additionalMakeVariables; + } + + /** + * Returns the execution path to the linker binary to use for this build. + * Relative paths are relative to the execution root. + */ + public PathFragment getLdExecutable() { + return ldExecutable; + } + + /** + * Returns the dynamic linking mode (full, off, or default). + */ + public DynamicMode getDynamicMode() { + return dynamicMode; + } + + /* + * If true then the directory name for non-LIPO targets will have a '-lipodata' suffix in + * AutoFDO mode. + */ + public boolean getAutoFdoLipoData() { + return cppOptions.autoFdoLipoData; + } + + /** + * Returns the STL label if given on the command line. {@code null} + * otherwise. + */ + public Label getStl() { + return cppOptions.stl; + } + + /* + * Returns the command-line "Make" variable overrides. + */ + @Override + public ImmutableMap<String, String> getCommandLineDefines() { + return commandLineDefines; + } + + /** + * Returns the command-line override value for the specified "Make" variable + * for this configuration, or null if none. + */ + public String getMakeVariableOverride(String var) { + return commandLineDefines.get(var); + } + + public boolean shouldScanIncludes() { + return cppOptions.scanIncludes; + } + + /** + * Returns the currently active LIPO compilation mode. + */ + public LipoMode getLipoMode() { + return cppOptions.lipoMode; + } + + public boolean isFdo() { + return cppOptions.isFdo(); + } + + public boolean isLipoOptimization() { + // The LIPO optimization bits are set in the LIPO context collector configuration, too. + return cppOptions.isLipoOptimization() && !isLipoContextCollector(); + } + + public boolean isLipoOptimizationOrInstrumentation() { + return cppOptions.isLipoOptimizationOrInstrumentation(); + } + + /** + * Returns true if it is AutoFDO LIPO build. + */ + public boolean isAutoFdoLipo() { + return cppOptions.fdoOptimize != null && FdoSupport.isAutoFdo(cppOptions.fdoOptimize) + && getLipoMode() != LipoMode.OFF; + } + + /** + * Returns the default header check mode. + */ + public HeadersCheckingMode getHeadersCheckingMode() { + return cppOptions.headersCheckingMode; + } + + /** + * Returns whether or not to strip the binaries. + */ + public boolean shouldStripBinaries() { + return stripBinaries; + } + + /** + * Returns the additional options to pass to strip when generating a + * {@code <name>.stripped} binary by this build. + */ + public List<String> getStripOpts() { + return cppOptions.stripoptList; + } + + /** + * Returns whether temporary outputs from gcc will be saved. + */ + public boolean getSaveTemps() { + return cppOptions.saveTemps; + } + + /** + * Returns the {@link PerLabelOptions} to apply to the gcc command line, if + * the label of the compiled file matches the regular expression. + */ + public List<PerLabelOptions> getPerFileCopts() { + return cppOptions.perFileCopts; + } + + public Label getLipoContextLabel() { + return cppOptions.getLipoContextLabel(); + } + + /** + * Returns the custom malloc library label. + */ + public Label customMalloc() { + return cppOptions.customMalloc; + } + + /** + * Returns the extra warnings enabled for C compilation. + */ + public List<String> getCWarns() { + return cppOptions.cWarns; + } + + /** + * Returns true if mostly-static C++ binaries should be skipped. + */ + public boolean skipStaticOutputs() { + return cppOptions.skipStaticOutputs; + } + + /** + * Returns true if Fission is specified for this build and supported by the crosstool. + */ + public boolean useFission() { + return cppOptions.fissionModes.contains(compilationMode) && supportsFission(); + } + + /** + * Returns true if all C++ compilations should produce position-independent code, links should + * produce position-independent executables, and dependencies with equivalent pre-built pic and + * nopic versions should apply the pic versions. Returns false if default settings should be + * applied (i.e. make no special provisions for pic code). + */ + public boolean forcePic() { + return cppOptions.forcePic; + } + + public boolean useStartEndLib() { + return cppOptions.useStartEndLib && supportsStartEndLib(); + } + + public boolean useThinArchives() { + return cppOptions.useThinArchives && supportsThinArchives(); + } + + /** + * Returns true if interface shared objects should be used. + */ + public boolean useInterfaceSharedObjects() { + return supportsInterfaceSharedObjects() && cppOptions.useInterfaceSharedObjects; + } + + public boolean forceIgnoreDashStatic() { + return cppOptions.forceIgnoreDashStatic; + } + + /** + * Returns true iff this build configuration requires inclusion extraction + * (for include scanning) in the action graph. + */ + public boolean needsIncludeScanning() { + return cppOptions.extractInclusions; + } + + public boolean createCppModuleMaps() { + return cppOptions.cppModuleMaps; + } + + /** + * Returns true if shared libraries must be compiled with position independent code + * on this platform or in this configuration. + */ + public boolean needsPic() { + return forcePic() || toolchainNeedsPic(); + } + + /** + * Returns true iff we should use ".pic.o" files when linking executables. + */ + public boolean usePicObjectsForBinaries() { + return forcePic() || usePicForBinaries(); + } + + public boolean legacyWholeArchive() { + return cppOptions.legacyWholeArchive; + } + + public boolean getSymbolCounts() { + return cppOptions.symbolCounts; + } + + public boolean getInmemoryDotdFiles() { + return cppOptions.inmemoryDotdFiles; + } + + public boolean useIsystemForIncludes() { + return cppOptions.useIsystemForIncludes; + } + + public LibcTop getLibcTop() { + return cppOptions.libcTop; + } + + public boolean getUseInterfaceSharedObjects() { + return cppOptions.useInterfaceSharedObjects; + } + + /** + * Returns the FDO support object. + */ + public FdoSupport getFdoSupport() { + return fdoSupport; + } + + /** + * Return the name of the directory (relative to the bin directory) that + * holds mangled links to shared libraries. This name is always set to + * the '{@code _solib_<cpu_archictecture_name>}. + */ + public String getSolibDirectory() { + return solibDirectory; + } + + /** + * Returns the path to the GNU binutils 'objcopy' binary to use for this + * build. (Corresponds to $(OBJCOPY) in make-dbg.) Relative paths are + * relative to the execution root. + */ + public PathFragment getObjCopyExecutable() { + return getToolPathFragment(CppConfiguration.Tool.OBJCOPY); + } + + /** + * Returns the path to the GNU binutils 'gcc' binary that should be used + * by this build. This binary should support compilation of both C (*.c) + * and C++ (*.cc) files. Relative paths are relative to the execution root. + */ + public PathFragment getCppExecutable() { + return getToolPathFragment(CppConfiguration.Tool.GCC); + } + + /** + * Returns the path to the GNU binutils 'g++' binary that should be used + * by this build. This binary should support linking of both C (*.c) + * and C++ (*.cc) files. Relative paths are relative to the execution root. + */ + public PathFragment getCppLinkExecutable() { + return getToolPathFragment(CppConfiguration.Tool.GCC); + } + + /** + * Returns the path to the GNU binutils 'cpp' binary that should be used + * by this build. Relative paths are relative to the execution root. + */ + public PathFragment getCpreprocessorExecutable() { + return getToolPathFragment(CppConfiguration.Tool.CPP); + } + + /** + * Returns the path to the GNU binutils 'gcov' binary that should be used + * by this build to analyze C++ coverage data. Relative paths are relative to + * the execution root. + */ + public PathFragment getGcovExecutable() { + return getToolPathFragment(CppConfiguration.Tool.GCOV); + } + + /** + * Returns the path to the 'gcov-tool' executable that should be used + * by this build. Relative paths are relative to the execution root. + */ + public PathFragment getGcovToolExecutable() { + return getToolPathFragment(CppConfiguration.Tool.GCOVTOOL); + } + + /** + * Returns the path to the GNU binutils 'nm' executable that should be used + * by this build. Used only for testing. Relative paths are relative to the + * execution root. + */ + public PathFragment getNmExecutable() { + return getToolPathFragment(CppConfiguration.Tool.NM); + } + + /** + * Returns the path to the GNU binutils 'objdump' executable that should be + * used by this build. Used only for testing. Relative paths are relative to + * the execution root. + */ + public PathFragment getObjdumpExecutable() { + return getToolPathFragment(CppConfiguration.Tool.OBJDUMP); + } + + /** + * Returns the path to the GNU binutils 'ar' binary to use for this build. + * Relative paths are relative to the execution root. + */ + public PathFragment getArExecutable() { + return getToolPathFragment(CppConfiguration.Tool.AR); + } + + /** + * Returns the path to the GNU binutils 'strip' executable that should be used + * by this build. Relative paths are relative to the execution root. + */ + public PathFragment getStripExecutable() { + return getToolPathFragment(CppConfiguration.Tool.STRIP); + } + + /** + * Returns the path to the GNU binutils 'dwp' binary that should be used by this + * build to combine debug info output from individual C++ compilations (i.e. .dwo + * files) into aggregate target-level debug packages. Relative paths are relative to the + * execution root. See https://gcc.gnu.org/wiki/DebugFission . + */ + public PathFragment getDwpExecutable() { + return getToolPathFragment(CppConfiguration.Tool.DWP); + } + + /** + * Returns the GNU System Name + */ + public String getTargetGnuSystemName() { + return targetSystemName; + } + + /** + * Returns the architecture component of the GNU System Name + */ + public String getGnuSystemArch() { + if (targetSystemName.indexOf('-') == -1) { + return targetSystemName; + } + return targetSystemName.substring(0, targetSystemName.indexOf('-')); + } + + /** + * Returns whether the configuration's purpose is only to collect LIPO-related data. + */ + public boolean isLipoContextCollector() { + return lipoContextCollector; + } + + @Override + public String getName() { + return "cpp"; + } + + @Override + public void reportInvalidOptions(EventHandler reporter, BuildOptions buildOptions) { + CppOptions cppOptions = buildOptions.get(CppOptions.class); + if (stripBinaries) { + boolean warn = cppOptions.coptList.contains("-g"); + for (PerLabelOptions opt : cppOptions.perFileCopts) { + warn |= opt.getOptions().contains("-g"); + } + if (warn) { + reporter.handle(Event.warn("Stripping enabled, but '--copt=-g' (or --per_file_copt=...@-g) " + + "specified. Debug information will be generated and then stripped away. This is " + + "probably not what you want! Use '-c dbg' for debug mode, or use '--strip=never' " + + "to disable stripping")); + } + } + + if (cppOptions.fdoInstrument != null && cppOptions.fdoOptimize != null) { + reporter.handle(Event.error("Cannot instrument and optimize for FDO at the same time. " + + "Remove one of the '--fdo_instrument' and '--fdo_optimize' options")); + } + + if (cppOptions.lipoContext != null) { + if (cppOptions.lipoMode != LipoMode.BINARY || cppOptions.fdoOptimize == null) { + reporter.handle(Event.warn("The --lipo_context option can only be used together with " + + "--fdo_optimize=<profile zip> and --lipo=binary. LIPO context will be ignored.")); + } + } else { + if (cppOptions.lipoMode == LipoMode.BINARY && cppOptions.fdoOptimize != null) { + reporter.handle(Event.error("The --lipo_context option must be specified when using " + + "--fdo_optimize=<profile zip> and --lipo=binary")); + } + } + if (cppOptions.lipoMode == LipoMode.BINARY && + compilationMode != CompilationMode.OPT) { + reporter.handle(Event.error( + "'--lipo=binary' can only be used with '--compilation_mode=opt' (or '-c opt')")); + } + + if (cppOptions.fissionModes.contains(compilationMode) && !supportsFission()) { + reporter.handle( + Event.warn("Fission is not supported by this crosstool. Please use a supporting " + + "crosstool to enable fission")); + } + } + + @Override + public void addGlobalMakeVariables(Builder<String, String> globalMakeEnvBuilder) { + // hardcoded CC->gcc setting for unit tests + globalMakeEnvBuilder.put("CC", getCppExecutable().getPathString()); + + // Make variables provided by crosstool/gcc compiler suite. + globalMakeEnvBuilder.put("AR", getArExecutable().getPathString()); + globalMakeEnvBuilder.put("NM", getNmExecutable().getPathString()); + globalMakeEnvBuilder.put("OBJCOPY", getObjCopyExecutable().getPathString()); + globalMakeEnvBuilder.put("STRIP", getStripExecutable().getPathString()); + + PathFragment gcovtool = getGcovToolExecutable(); + if (gcovtool != null) { + // gcov-tool is optional in Crosstool + globalMakeEnvBuilder.put("GCOVTOOL", gcovtool.getPathString()); + } + + if (getTargetLibc().startsWith("glibc-")) { + globalMakeEnvBuilder.put("GLIBC_VERSION", + getTargetLibc().substring("glibc-".length())); + } else { + globalMakeEnvBuilder.put("GLIBC_VERSION", getTargetLibc()); + } + + globalMakeEnvBuilder.put("C_COMPILER", getCompiler()); + globalMakeEnvBuilder.put("TARGET_CPU", getTargetCpu()); + + // Deprecated variables + + // TODO(bazel-team): (2009) These variables are so rarely used we should try to eliminate + // them entirely. see: "cs -f=BUILD -noi GNU_TARGET" and "cs -f=build_defs -noi + // GNU_TARGET" + globalMakeEnvBuilder.put("CROSSTOOLTOP", crosstoolTopPathFragment.getPathString()); + globalMakeEnvBuilder.put("GLIBC", getTargetLibc()); + globalMakeEnvBuilder.put("GNU_TARGET", targetSystemName); + + globalMakeEnvBuilder.putAll(getAdditionalMakeVariables()); + + globalMakeEnvBuilder.put("ABI_GLIBC_VERSION", getAbiGlibcVersion()); + globalMakeEnvBuilder.put("ABI", abi); + } + + @Override + public void addImplicitLabels(Multimap<String, Label> implicitLabels) { + if (getLibcLabel() != null) { + implicitLabels.put("crosstool", getLibcLabel()); + } + + implicitLabels.put("crosstool", crosstoolTop); + } + + @Override + public void prepareHook(Path execRoot, ArtifactFactory artifactFactory, PathFragment genfilesPath, + PackageRootResolver resolver) throws ViewCreationFailedException { + try { + getFdoSupport().prepareToBuild(execRoot, genfilesPath, artifactFactory, resolver); + } catch (ZipException e) { + throw new ViewCreationFailedException("Error reading provided FDO zip file", e); + } catch (FdoException | IOException e) { + throw new ViewCreationFailedException("Error while initializing FDO support", e); + } + } + + @Override + public void declareSkyframeDependencies(Environment env) { + getFdoSupport().declareSkyframeDependencies(env, execRoot); + } + + @Override + public void addRoots(List<Root> roots) { + // Fdo root can only exist for the target configuration. + FdoSupport fdoSupport = getFdoSupport(); + if (fdoSupport.getFdoRoot() != null) { + roots.add(fdoSupport.getFdoRoot()); + } + + // Grepped header includes; this root is not configuration specific. + roots.add(getGreppedIncludesDirectory()); + } + + @Override + public Map<String, String> getCoverageEnvironment() { + ImmutableMap.Builder<String, String> env = ImmutableMap.builder(); + env.put("COVERAGE_GCOV_PATH", getGcovExecutable().getPathString()); + PathFragment fdoInstrument = getFdoSupport().getFdoInstrument(); + if (fdoInstrument != null) { + env.put("FDO_DIR", fdoInstrument.getPathString()); + } + return env.build(); + } + + @Override + public ImmutableList<Label> getCoverageLabels() { + // TODO(bazel-team): Using a gcov-specific crosstool filegroup here could reduce the number of + // inputs significantly. We'd also need to add logic in tools/coverage/collect_coverage.sh to + // drop crosstool dependency if metadataFiles does not contain *.gcno artifacts. + return ImmutableList.of(crosstoolTop); + } + + @Override + public String getOutputDirectoryName() { + String lipoSuffix; + if (getLipoMode() != LipoMode.OFF && !isAutoFdoLipo()) { + lipoSuffix = "-lipo"; + } else if (getAutoFdoLipoData()) { + lipoSuffix = "-lipodata"; + } else { + lipoSuffix = ""; + } + return toolchainIdentifier + lipoSuffix; + } + + @Override + public String getConfigurationNameSuffix() { + return isLipoContextCollector() ? "collector" : null; + } + + @Override + public String getPlatformName() { + return getToolchainIdentifier(); + } + + @Override + public boolean supportsIncrementalBuild() { + return !isLipoOptimization(); + } + + @Override + public boolean performsStaticLink() { + return getLinkOptions().contains("-static"); + } + + /** + * Returns true if we should share identical native libraries between different targets. + */ + public boolean shareNativeDeps() { + return cppOptions.shareNativeDeps; + } + + @Override + public void prepareForExecutionPhase() throws IOException { + // _fdo has a prefix of "_", but it should nevertheless be deleted. Detailed description + // of the structure of the symlinks / directories can be found at FdoSupport.extractFdoZip(). + // We actually create a directory named "blaze-fdo" under the exec root, the previous version + // of which is deleted in FdoSupport.prepareToBuildExec(). We cannot do that just before the + // execution phase because that needs to happen before the analysis phase (in order to create + // the artifacts corresponding to the .gcda files). + Path tempPath = execRoot.getRelative("_fdo"); + if (tempPath.exists()) { + FileSystemUtils.deleteTree(tempPath); + } + } + + @Override + public Map<String, Object> lateBoundOptionDefaults() { + // --cpu defaults to null. With that default, the actual target cpu string gets picked up + // by the "default_target_cpu" crosstool parameter. + return ImmutableMap.<String, Object>of("cpu", getTargetCpu()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfigurationLoader.java new file mode 100644 index 0000000000..de20283419 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfigurationLoader.java @@ -0,0 +1,174 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.RedirectChaser; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig; + +import javax.annotation.Nullable; + +/** + * Loader for C++ configurations. + */ +public class CppConfigurationLoader implements ConfigurationFragmentFactory { + @Override + public Class<? extends Fragment> creates() { + return CppConfiguration.class; + } + + private final Function<String, String> cpuTransformer; + + /** + * Creates a new CrosstoolConfigurationLoader instance with the given + * configuration provider. The configuration provider is used to perform + * caller-specific configuration file lookup. + */ + public CppConfigurationLoader(Function<String, String> cpuTransformer) { + this.cpuTransformer = cpuTransformer; + } + + @Override + public CppConfiguration create(ConfigurationEnvironment env, BuildOptions options) + throws InvalidConfigurationException { + CppConfigurationParameters params = createParameters(env, options); + if (params == null) { + return null; + } + return new CppConfiguration(params); + } + + /** + * Value class for all the data needed to create a {@link CppConfiguration}. + */ + public static class CppConfigurationParameters { + protected final CrosstoolConfig.CToolchain toolchain; + protected final String cacheKeySuffix; + protected final BuildOptions buildOptions; + protected final Label crosstoolTop; + protected final Label ccToolchainLabel; + protected final Path fdoZip; + protected final Path execRoot; + + CppConfigurationParameters(CrosstoolConfig.CToolchain toolchain, + String cacheKeySuffix, + BuildOptions buildOptions, + Path fdoZip, + Path execRoot, + Label crosstoolTop, + Label ccToolchainLabel) { + this.toolchain = toolchain; + this.cacheKeySuffix = cacheKeySuffix; + this.buildOptions = buildOptions; + this.fdoZip = fdoZip; + this.execRoot = execRoot; + this.crosstoolTop = crosstoolTop; + this.ccToolchainLabel = ccToolchainLabel; + } + } + + @Nullable + protected CppConfigurationParameters createParameters( + ConfigurationEnvironment env, BuildOptions options) throws InvalidConfigurationException { + BlazeDirectories directories = env.getBlazeDirectories(); + if (directories == null) { + return null; + } + Label crosstoolTop = RedirectChaser.followRedirects(env, + options.get(CppOptions.class).crosstoolTop, "crosstool_top"); + if (crosstoolTop == null) { + return null; + } + CrosstoolConfigurationLoader.CrosstoolFile file = + CrosstoolConfigurationLoader.readCrosstool(env, crosstoolTop); + if (file == null) { + return null; + } + CrosstoolConfig.CToolchain toolchain = + CrosstoolConfigurationLoader.selectToolchain(file.getProto(), options, cpuTransformer); + + // FDO + // TODO(bazel-team): move this to CppConfiguration.prepareHook + CppOptions cppOptions = options.get(CppOptions.class); + Path fdoZip; + if (cppOptions.fdoOptimize == null) { + fdoZip = null; + } else if (cppOptions.fdoOptimize.startsWith("//")) { + try { + Target target = env.getTarget(Label.parseAbsolute(cppOptions.fdoOptimize)); + if (target == null) { + return null; + } + if (!(target instanceof InputFile)) { + throw new InvalidConfigurationException( + "--fdo_optimize cannot accept targets that do not refer to input files"); + } + fdoZip = env.getPath(target.getPackage(), target.getName()); + if (fdoZip == null) { + throw new InvalidConfigurationException( + "The --fdo_optimize parameter you specified resolves to a file that does not exist"); + } + } catch (NoSuchPackageException | NoSuchTargetException | SyntaxException e) { + throw new InvalidConfigurationException(e); + } + } else { + fdoZip = directories.getWorkspace().getRelative(cppOptions.fdoOptimize); + } + + Label ccToolchainLabel; + try { + ccToolchainLabel = crosstoolTop.getRelative("cc-compiler-" + toolchain.getTargetCpu()); + } catch (Label.SyntaxException e) { + throw new InvalidConfigurationException(String.format( + "'%s' is not a valid CPU. It should only consist of characters valid in labels", + toolchain.getTargetCpu())); + } + + Target ccToolchain; + try { + ccToolchain = env.getTarget(ccToolchainLabel); + if (ccToolchain == null) { + return null; + } + } catch (NoSuchThingException e) { + throw new InvalidConfigurationException(String.format( + "The toolchain rule '%s' does not exist", ccToolchainLabel)); + } + + if (!(ccToolchain instanceof Rule) + || !((Rule) ccToolchain).getRuleClass().equals("cc_toolchain")) { + throw new InvalidConfigurationException(String.format( + "The label '%s' is not a cc_toolchain rule", ccToolchainLabel)); + } + + return new CppConfigurationParameters(toolchain, file.getMd5(), options, + fdoZip, directories.getExecRoot(), crosstoolTop, ccToolchainLabel); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugFileProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugFileProvider.java new file mode 100644 index 0000000000..c0fcb11330 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugFileProvider.java @@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A target that provides .dwo files which can be combined into a .dwp packaging step. See + * https://gcc.gnu.org/wiki/DebugFission for details. + */ +@Immutable +public final class CppDebugFileProvider implements TransitiveInfoProvider { + + private final NestedSet<Artifact> transitiveDwoFiles; + private final NestedSet<Artifact> transitivePicDwoFiles; + + public CppDebugFileProvider(NestedSet<Artifact> transitiveDwoFiles, + NestedSet<Artifact> transitivePicDwoFiles) { + this.transitiveDwoFiles = transitiveDwoFiles; + this.transitivePicDwoFiles = transitivePicDwoFiles; + } + + /** + * Returns the .dwo files that should be included in this target's .dwp packaging (if this + * target is linked) or passed through to a dependant's .dwp packaging (e.g. if this is a + * cc_library depended on by a statically linked cc_binary). + * + * Assumes the corresponding link consumes .o files (vs. .pic.o files). + */ + public NestedSet<Artifact> getTransitiveDwoFiles() { + return transitiveDwoFiles; + } + + /** + * Same as above, but assumes the corresponding link consumes pic.o files. + */ + public NestedSet<Artifact> getTransitivePicDwoFiles() { + return transitivePicDwoFiles; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugPackageProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugPackageProvider.java new file mode 100644 index 0000000000..864a4d5a11 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugPackageProvider.java @@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import javax.annotation.Nullable; + +/** + * Provides the binary artifact and its associated .dwp files, if fission is enabled. + * If Fission ({@link https://gcc.gnu.org/wiki/DebugFission}) is not enabled, the + * dwp file will be null. + */ +@Immutable +public final class CppDebugPackageProvider implements TransitiveInfoProvider { + + private final Artifact strippedArtifact; + private final Artifact unstrippedArtifact; + @Nullable private final Artifact dwpArtifact; + + public CppDebugPackageProvider( + Artifact strippedArtifact, + Artifact unstrippedArtifact, + @Nullable Artifact dwpArtifact) { + Preconditions.checkNotNull(strippedArtifact); + Preconditions.checkNotNull(unstrippedArtifact); + this.strippedArtifact = strippedArtifact; + this.unstrippedArtifact = unstrippedArtifact; + this.dwpArtifact = dwpArtifact; + } + + /** + * Returns the stripped file (the explicit ".stripped" target). + */ + public final Artifact getStrippedArtifact() { + return strippedArtifact; + } + + /** + * Returns the unstripped file (the default executable target). + */ + public final Artifact getUnstrippedArtifact() { + return unstrippedArtifact; + } + + /** + * Returns the .dwp file (for fission builds) or null if --fission=no. + */ + @Nullable + public final Artifact getDwpArtifact() { + return dwpArtifact; + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java new file mode 100644 index 0000000000..d9bb7b6803 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java @@ -0,0 +1,141 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.util.FileType; + +import java.util.List; +import java.util.regex.Pattern; + +/** + * C++-related file type definitions. + */ +public final class CppFileTypes { + public static final FileType CPP_SOURCE = FileType.of(".cc", ".cpp", ".cxx", ".C"); + public static final FileType C_SOURCE = FileType.of(".c"); + public static final FileType CPP_HEADER = FileType.of(".h", ".hh", ".hpp", ".hxx", ".inc"); + public static final FileType CPP_TEXTUAL_INCLUDE = FileType.of(".inc"); + + public static final FileType PIC_PREPROCESSED_C = FileType.of(".pic.i"); + public static final FileType PREPROCESSED_C = new FileType() { + final String ext = ".i"; + @Override + public boolean apply(String filename) { + return filename.endsWith(ext) && !PIC_PREPROCESSED_C.matches(filename); + } + @Override + public List<String> getExtensions() { + return ImmutableList.of(ext); + } + }; + public static final FileType PIC_PREPROCESSED_CPP = FileType.of(".pic.ii"); + public static final FileType PREPROCESSED_CPP = new FileType() { + final String ext = ".ii"; + @Override + public boolean apply(String filename) { + return filename.endsWith(ext) && !PIC_PREPROCESSED_CPP.matches(filename); + } + @Override + public List<String> getExtensions() { + return ImmutableList.of(ext); + } + }; + + public static final FileType ASSEMBLER_WITH_C_PREPROCESSOR = FileType.of(".S"); + public static final FileType PIC_ASSEMBLER = FileType.of(".pic.s"); + public static final FileType ASSEMBLER = new FileType() { + final String ext = ".s"; + @Override + public boolean apply(String filename) { + return filename.endsWith(ext) && !PIC_ASSEMBLER.matches(filename); + } + @Override + public List<String> getExtensions() { + return ImmutableList.of(ext); + } + }; + + public static final FileType PIC_ARCHIVE = FileType.of(".pic.a"); + public static final FileType ARCHIVE = new FileType() { + final String ext = ".a"; + @Override + public boolean apply(String filename) { + return filename.endsWith(ext) && !PIC_ARCHIVE.matches(filename); + } + @Override + public List<String> getExtensions() { + return ImmutableList.of(ext); + } + }; + + public static final FileType ALWAYS_LINK_PIC_LIBRARY = FileType.of(".pic.lo"); + public static final FileType ALWAYS_LINK_LIBRARY = new FileType() { + final String ext = ".lo"; + @Override + public boolean apply(String filename) { + return filename.endsWith(ext) && !ALWAYS_LINK_PIC_LIBRARY.matches(filename); + } + @Override + public List<String> getExtensions() { + return ImmutableList.of(ext); + } + }; + + public static final FileType PIC_OBJECT_FILE = FileType.of(".pic.o"); + public static final FileType OBJECT_FILE = new FileType() { + final String ext = ".o"; + @Override + public boolean apply(String filename) { + return filename.endsWith(ext) && !PIC_OBJECT_FILE.matches(filename); + } + @Override + public List<String> getExtensions() { + return ImmutableList.of(ext); + } + }; + + + public static final FileType SHARED_LIBRARY = FileType.of(".so"); + public static final FileType INTERFACE_SHARED_LIBRARY = FileType.of(".ifso"); + public static final FileType LINKER_SCRIPT = FileType.of(".lds"); + // Matches shared libraries with version names in the extension, i.e. + // libmylib.so.2 or libmylib.so.2.10. + private static final Pattern VERSIONED_SHARED_LIBRARY_PATTERN = + Pattern.compile("^.+\\.so(\\.\\d+)+$"); + public static final FileType VERSIONED_SHARED_LIBRARY = new FileType() { + @Override + public boolean apply(String filename) { + // Because regex matching can be slow, we first do a quick digit check on the final + // character before risking the full-on regex match. This should eliminate the performance + // hit on practically every non-qualifying file type. + if (Character.isDigit(filename.charAt(filename.length() - 1))) { + return VERSIONED_SHARED_LIBRARY_PATTERN.matcher(filename).matches(); + } else { + return false; + } + } + }; + + public static final FileType COVERAGE_NOTES = FileType.of(".gcno"); + public static final FileType COVERAGE_DATA = FileType.of(".gcda"); + public static final FileType COVERAGE_DATA_IMPORTS = FileType.of(".gcda.imports"); + public static final FileType GCC_AUTO_PROFILE = FileType.of(".afdo"); + + public static final FileType CPP_MODULE_MAP = FileType.of(".cppmap"); + public static final FileType CPP_MODULE = FileType.of(".pcm"); + + // Output of the dwp tool + public static final FileType DEBUG_INFO_PACKAGE = FileType.of(".dwp"); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java new file mode 100644 index 0000000000..5bc1363072 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java @@ -0,0 +1,529 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.MiddlemanFactory; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.Util; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CcLinkParams.Linkstamp; +import com.google.devtools.build.lib.rules.cpp.CppCompilationContext.Builder; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.shell.ShellUtils; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.util.IncludeScanningUtil; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Helper class for functionality shared by cpp related rules. + * + * <p>This class can be used only after the loading phase. + */ +public class CppHelper { + // TODO(bazel-team): should this use Link.SHARED_LIBRARY_FILETYPES? + public static final FileTypeSet SHARED_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY); + + private static final FileTypeSet CPP_FILETYPES = FileTypeSet.of( + CppFileTypes.CPP_HEADER, + CppFileTypes.CPP_SOURCE); + + private CppHelper() { + // prevents construction + } + + /** + * Merges the STL and toolchain contexts into context builder. The STL is automatically determined + * using the ":stl" attribute. + */ + public static void mergeToolchainDependentContext(RuleContext ruleContext, + Builder contextBuilder) { + TransitiveInfoCollection stl = ruleContext.getPrerequisite(":stl", Mode.TARGET); + if (stl != null) { + // TODO(bazel-team): Clean this up. + contextBuilder.addSystemIncludeDir(stl.getLabel().getPackageFragment().getRelative("gcc3")); + contextBuilder.mergeDependentContext(stl.getProvider(CppCompilationContext.class)); + } + CcToolchainProvider toolchain = getToolchain(ruleContext); + if (toolchain != null) { + contextBuilder.mergeDependentContext(toolchain.getCppCompilationContext()); + } + } + + /** + * Returns the malloc implementation for the given target. + */ + public static TransitiveInfoCollection mallocForTarget(RuleContext ruleContext) { + if (ruleContext.getFragment(CppConfiguration.class).customMalloc() != null) { + return ruleContext.getPrerequisite(":default_malloc", Mode.TARGET); + } else { + return ruleContext.getPrerequisite("malloc", Mode.TARGET); + } + } + + /** + * Expands Make variables in a list of string and tokenizes the result. If the package feature + * no_copts_tokenization is set, tokenize only items consisting of a single make variable. + * + * @param ruleContext the ruleContext to be used as the context of Make variable expansion + * @param attributeName the name of the attribute to use in error reporting + * @param input the list of strings to expand + * @return a list of strings containing the expanded and tokenized values for the + * attribute + */ + // TODO(bazel-team): Move to CcCommon; refactor CcPlugin to use either CcLibraryHelper or + // CcCommon. + static List<String> expandMakeVariables( + RuleContext ruleContext, String attributeName, List<String> input) { + boolean tokenization = + !ruleContext.getFeatures().contains("no_copts_tokenization"); + + List<String> tokens = new ArrayList<>(); + for (String token : input) { + try { + // Legacy behavior: tokenize all items. + if (tokenization) { + ruleContext.tokenizeAndExpandMakeVars(tokens, attributeName, token); + } else { + String exp = ruleContext.expandSingleMakeVariable(attributeName, token); + if (exp != null) { + ShellUtils.tokenize(tokens, exp); + } else { + tokens.add(ruleContext.expandMakeVariables(attributeName, token)); + } + } + } catch (ShellUtils.TokenizationException e) { + ruleContext.attributeError(attributeName, e.getMessage()); + } + } + return ImmutableList.copyOf(tokens); + } + + /** + * Appends the tokenized values of the copts attribute to copts. + */ + public static ImmutableList<String> getAttributeCopts(RuleContext ruleContext, String attr) { + Preconditions.checkArgument(ruleContext.getRule().isAttrDefined(attr, Type.STRING_LIST)); + List<String> unexpanded = ruleContext.attributes().get(attr, Type.STRING_LIST); + + return ImmutableList.copyOf(expandMakeVariables(ruleContext, attr, unexpanded)); + } + + /** + * Expands attribute value either using label expansion + * (if attemptLabelExpansion == {@code true} and it does not look like make + * variable or flag) or tokenizes and expands make variables. + */ + public static void expandAttribute(RuleContext ruleContext, + List<String> values, String attrName, String attrValue, boolean attemptLabelExpansion) { + if (attemptLabelExpansion && CppHelper.isLinkoptLabel(attrValue)) { + if (!CppHelper.expandLabel(ruleContext, values, attrValue)) { + ruleContext.attributeError(attrName, "could not resolve label '" + attrValue + "'"); + } + } else { + ruleContext.tokenizeAndExpandMakeVars(values, attrName, attrValue); + } + } + + /** + * Determines if a linkopt can be a label. Linkopts come in 2 varieties: + * literals -- flags like -Xl and makefile vars like $(LD) -- and labels, + * which we should expand into filenames. + * + * @param linkopt the link option to test. + * @return true if the linkopt is not a flag (starting with "-") or a makefile + * variable (starting with "$"); + */ + private static boolean isLinkoptLabel(String linkopt) { + return !linkopt.startsWith("$") && !linkopt.startsWith("-"); + } + + /** + * Expands a label against the target's deps, adding the expanded path strings + * to the linkopts. + * + * @param linkopts the linkopts to add the expanded label to + * @param labelName the name of the label to expand + * @return true if the label was expanded successfully, false otherwise + */ + private static boolean expandLabel(RuleContext ruleContext, List<String> linkopts, + String labelName) { + try { + Label label = ruleContext.getLabel().getRelative(labelName); + for (FileProvider target : ruleContext + .getPrerequisites("deps", Mode.TARGET, FileProvider.class)) { + if (target.getLabel().equals(label)) { + for (Artifact artifact : target.getFilesToBuild()) { + linkopts.add(artifact.getExecPathString()); + } + return true; + } + } + } catch (SyntaxException e) { + // Quietly ignore and fall through. + } + linkopts.add(labelName); + return false; + } + + /** + * This almost trivial method looks up the :cc_toolchain attribute on the rule context, makes sure + * that it refers to a rule that has a {@link CcToolchainProvider} (gives an error otherwise), and + * returns a reference to that {@link CcToolchainProvider}. The method only returns {@code null} + * if there is no such attribute (this is currently not an error). + */ + @Nullable public static CcToolchainProvider getToolchain(RuleContext ruleContext) { + if (ruleContext.attributes().getAttributeDefinition(":cc_toolchain") == null) { + // TODO(bazel-team): Report an error or throw an exception in this case. + return null; + } + TransitiveInfoCollection dep = ruleContext.getPrerequisite(":cc_toolchain", Mode.TARGET); + return getToolchain(ruleContext, dep); + } + + /** + * This almost trivial method makes sure that the given info collection has a {@link + * CcToolchainProvider} (gives an error otherwise), and returns a reference to that {@link + * CcToolchainProvider}. The method never returns {@code null}, even if there is no toolchain. + */ + public static CcToolchainProvider getToolchain(RuleContext ruleContext, + TransitiveInfoCollection dep) { + // TODO(bazel-team): Consider checking this generally at the attribute level. + if ((dep == null) || (dep.getProvider(CcToolchainProvider.class) == null)) { + ruleContext.ruleError("The selected C++ toolchain is not a cc_toolchain rule"); + return CcToolchainProvider.EMPTY_TOOLCHAIN_IS_ERROR; + } + return dep.getProvider(CcToolchainProvider.class); + } + + /** + * Returns the directory where object files are created. + */ + public static PathFragment getObjDirectory(Label ruleLabel) { + return AnalysisUtils.getUniqueDirectory(ruleLabel, new PathFragment("_objs")); + } + + /** + * Creates a grep-includes ExtractInclusions action for generated sources/headers in the + * needsIncludeScanning() BuildConfiguration case. Returns a map from original header + * Artifact to the output Artifact of grepping over it. The return value only includes + * entries for generated sources or headers when --extract_generated_inclusions is enabled. + * + * <p>Previously, incremental rebuilds redid all include scanning work + * for a given .cc source in serial. For high-latency file systems, this could cause + * performance problems if many headers are generated. + */ + @Nullable + public static final Map<Artifact, Artifact> createExtractInclusions(RuleContext ruleContext, + Iterable<Artifact> prerequisites) { + Map<Artifact, Artifact> extractions = new HashMap<>(); + for (Artifact prerequisite : prerequisites) { + Artifact scanned = createExtractInclusions(ruleContext, prerequisite); + if (scanned != null) { + extractions.put(prerequisite, scanned); + } + } + return extractions; + } + + /** + * Creates a grep-includes ExtractInclusions action for generated sources/headers in the + * needsIncludeScanning() BuildConfiguration case. + * + * <p>Previously, incremental rebuilds redid all include scanning work for a given + * .cc source in serial. For high-latency file systems, this could cause + * performance problems if many headers are generated. + */ + private static final Artifact createExtractInclusions(RuleContext ruleContext, + Artifact prerequisite) { + if (ruleContext != null && + ruleContext.getFragment(CppConfiguration.class).needsIncludeScanning() && + !prerequisite.isSourceArtifact() && + CPP_FILETYPES.matches(prerequisite.getFilename())) { + Artifact scanned = getIncludesOutput(ruleContext, prerequisite); + ruleContext.registerAction( + new ExtractInclusionAction(ruleContext.getActionOwner(), prerequisite, scanned)); + return scanned; + } + return null; + } + + private static Artifact getIncludesOutput(RuleContext ruleContext, Artifact src) { + Root root = ruleContext.getFragment(CppConfiguration.class).getGreppedIncludesDirectory(); + PathFragment relOut = IncludeScanningUtil.getRootRelativeOutputPath(src.getExecPath()); + return ruleContext.getAnalysisEnvironment().getDerivedArtifact(relOut, root); + } + + /** + * Returns the workspace-relative filename for the linked artifact. + */ + public static PathFragment getLinkedFilename(RuleContext ruleContext, + LinkTargetType linkType) { + PathFragment relativePath = Util.getWorkspaceRelativePath(ruleContext.getTarget()); + PathFragment linkedFileName = (linkType == LinkTargetType.EXECUTABLE) ? + relativePath : + relativePath.replaceName("lib" + relativePath.getBaseName() + linkType.getExtension()); + return linkedFileName; + } + + /** + * Resolves the linkstamp collection from the {@code CcLinkParams} into a map. + * + * <p>Emits a warning on the rule if there are identical linkstamp artifacts with different + * compilation contexts. + */ + public static Map<Artifact, ImmutableList<Artifact>> resolveLinkstamps(RuleContext ruleContext, + CcLinkParams linkParams) { + Map<Artifact, ImmutableList<Artifact>> result = new LinkedHashMap<>(); + for (Linkstamp pair : linkParams.getLinkstamps()) { + Artifact artifact = pair.getArtifact(); + if (result.containsKey(artifact)) { + ruleContext.ruleWarning("rule inherits the '" + artifact.toDetailString() + + "' linkstamp file from more than one cc_library rule"); + } + result.put(artifact, pair.getDeclaredIncludeSrcs()); + } + return result; + } + + public static void addTransitiveLipoInfoForCommonAttributes( + RuleContext ruleContext, + CcCompilationOutputs outputs, + NestedSetBuilder<IncludeScannable> scannableBuilder) { + + TransitiveLipoInfoProvider stl = null; + if (ruleContext.getRule().getAttributeDefinition(":stl") != null && + ruleContext.getPrerequisite(":stl", Mode.TARGET) != null) { + // If the attribute is defined, it is never null. + stl = ruleContext.getPrerequisite(":stl", Mode.TARGET) + .getProvider(TransitiveLipoInfoProvider.class); + } + if (stl != null) { + scannableBuilder.addTransitive(stl.getTransitiveIncludeScannables()); + } + + for (TransitiveLipoInfoProvider dep : + ruleContext.getPrerequisites("deps", Mode.TARGET, TransitiveLipoInfoProvider.class)) { + scannableBuilder.addTransitive(dep.getTransitiveIncludeScannables()); + } + + if (ruleContext.getRule().getRuleClassObject().hasAttr("malloc", Type.LABEL)) { + TransitiveInfoCollection malloc = mallocForTarget(ruleContext); + TransitiveLipoInfoProvider provider = malloc.getProvider(TransitiveLipoInfoProvider.class); + if (provider != null) { + scannableBuilder.addTransitive(provider.getTransitiveIncludeScannables()); + } + } + + for (IncludeScannable scannable : outputs.getLipoScannables()) { + Preconditions.checkState(scannable.getIncludeScannerSources().size() == 1); + scannableBuilder.add(scannable); + } + } + + // TODO(bazel-team): figure out a way to merge these 2 methods. See the Todo in + // CcCommonConfiguredTarget.noCoptsMatches(). + /** + * Determines if we should apply -fPIC for this rule's C++ compilations. This determination + * is generally made by the global C++ configuration settings "needsPic" and + * and "usePicForBinaries". However, an individual rule may override these settings by applying + * -fPIC" to its "nocopts" attribute. This allows incompatible rules to "opt out" of global PIC + * settings (see bug: "Provide a way to turn off -fPIC for targets that can't be built that way"). + * + * @param ruleContext the context of the rule to check + * @param forBinary true if compiling for a binary, false if for a shared library + * @return true if this rule's compilations should apply -fPIC, false otherwise + */ + public static boolean usePic(RuleContext ruleContext, boolean forBinary) { + if (CcCommon.noCoptsMatches("-fPIC", ruleContext)) { + return false; + } + CppConfiguration config = ruleContext.getFragment(CppConfiguration.class); + return forBinary ? config.usePicObjectsForBinaries() : config.needsPic(); + } + + /** + * Returns the LIPO context provider for configured target, + * or null if such a provider doesn't exist. + */ + public static LipoContextProvider getLipoContextProvider(RuleContext ruleContext) { + if (ruleContext.getRule().getAttributeDefinition(":lipo_context_collector") == null) { + return null; + } + + TransitiveInfoCollection dep = + ruleContext.getPrerequisite(":lipo_context_collector", Mode.DONT_CHECK); + return (dep != null) ? dep.getProvider(LipoContextProvider.class) : null; + } + + // Creates CppModuleMap object, and adds it to C++ compilation context. + public static CppModuleMap addCppModuleMapToContext(RuleContext ruleContext, + CppCompilationContext.Builder contextBuilder) { + if (!ruleContext.getFragment(CppConfiguration.class).createCppModuleMaps()) { + return null; + } + if (getToolchain(ruleContext).getCppCompilationContext().getCppModuleMap() == null) { + return null; + } + // Create the module map artifact as a genfile. + PathFragment mapPath = FileSystemUtils.appendExtension(ruleContext.getLabel().toPathFragment(), + Iterables.getOnlyElement(CppFileTypes.CPP_MODULE_MAP.getExtensions())); + Artifact mapFile = ruleContext.getAnalysisEnvironment().getDerivedArtifact(mapPath, + ruleContext.getConfiguration().getGenfilesDirectory()); + CppModuleMap moduleMap = + new CppModuleMap(mapFile, ruleContext.getLabel().toString()); + contextBuilder.setCppModuleMap(moduleMap); + return moduleMap; + } + + /** + * Returns a middleman for all files to build for the given configured target, + * substituting shared library artifacts with corresponding solib symlinks. If + * multiple calls are made, then it returns the same artifact for configurations + * with the same internal directory. + * + * <p>The resulting middleman only aggregates the inputs and must be expanded + * before populating the set of files necessary to execute an action. + */ + static List<Artifact> getAggregatingMiddlemanForCppRuntimes(RuleContext ruleContext, + String purpose, TransitiveInfoCollection dep, String solibDirOverride, + BuildConfiguration configuration) { + return getMiddlemanInternal( + ruleContext.getAnalysisEnvironment(), ruleContext, ruleContext.getActionOwner(), purpose, + dep, true, true, solibDirOverride, configuration); + } + + @VisibleForTesting + public static List<Artifact> getAggregatingMiddlemanForTesting(AnalysisEnvironment env, + RuleContext ruleContext, ActionOwner owner, String purpose, TransitiveInfoCollection dep, + boolean useSolibSymlinks, BuildConfiguration configuration) { + return getMiddlemanInternal( + env, ruleContext, owner, purpose, dep, useSolibSymlinks, false, null, configuration); + } + + /** + * Internal implementation for getAggregatingMiddlemanForCppRuntimes. + */ + private static List<Artifact> getMiddlemanInternal(AnalysisEnvironment env, + RuleContext ruleContext, ActionOwner actionOwner, String purpose, + TransitiveInfoCollection dep, boolean useSolibSymlinks, boolean isCppRuntime, + String solibDirOverride, BuildConfiguration configuration) { + if (dep == null) { + return ImmutableList.of(); + } + MiddlemanFactory factory = env.getMiddlemanFactory(); + Iterable<Artifact> artifacts = dep.getProvider(FileProvider.class).getFilesToBuild(); + if (useSolibSymlinks) { + List<Artifact> symlinkedArtifacts = new ArrayList<>(); + for (Artifact artifact : artifacts) { + symlinkedArtifacts.add(solibArtifactMaybe( + ruleContext, artifact, isCppRuntime, solibDirOverride, configuration)); + } + artifacts = symlinkedArtifacts; + purpose += "_with_solib"; + } + return ImmutableList.of(factory.createMiddlemanAllowMultiple( + env, actionOwner, purpose, artifacts, configuration.getMiddlemanDirectory())); + } + + /** + * If the artifact is a shared library, returns the solib symlink artifact associated with it. + * + * @param ruleContext the context of the rule that creates the symlink + * @param artifact the library the solib symlink should point to + * @param isCppRuntime whether the library is a C++ runtime + * @param solibDirOverride if not null, forces the solib symlink to be in this directory + */ + private static Artifact solibArtifactMaybe(RuleContext ruleContext, Artifact artifact, + boolean isCppRuntime, String solibDirOverride, BuildConfiguration configuration) { + if (SHARED_LIBRARY_FILETYPES.matches(artifact.getFilename())) { + return isCppRuntime + ? SolibSymlinkAction.getCppRuntimeSymlink( + ruleContext, artifact, solibDirOverride, configuration) + .getArtifact() + : SolibSymlinkAction.getDynamicLibrarySymlink( + ruleContext, artifact, false, true, configuration) + .getArtifact(); + } else { + return artifact; + } + } + + /** + * Returns the type of archives being used. + */ + public static Link.ArchiveType archiveType(BuildConfiguration config) { + CppConfiguration cppConfig = config.getFragment(CppConfiguration.class); + return cppConfig.archiveType(); + } + + /** + * Returns the FDO build subtype. + */ + public static String getFdoBuildStamp(CppConfiguration cppConfiguration) { + if (cppConfiguration.getFdoSupport().isAutoFdoEnabled()) { + return (cppConfiguration.getLipoMode() == LipoMode.BINARY) ? "ALIPO" : "AFDO"; + } + if (cppConfiguration.isFdo()) { + return (cppConfiguration.getLipoMode() == LipoMode.BINARY) ? "LIPO" : "FDO"; + } + return null; + } + + /** + * Returns a relative path to the bin directory for data in AutoFDO LIPO mode. + */ + public static PathFragment getLipoDataBinFragment(BuildConfiguration configuration) { + PathFragment parent = configuration.getBinFragment().getParentDirectory(); + return parent.replaceName(parent.getBaseName() + "-lipodata") + .getChild(configuration.getBinFragment().getBaseName()); + } + + /** + * Returns a relative path to the genfiles directory for data in AutoFDO LIPO mode. + */ + public static PathFragment getLipoDataGenfilesFragment(BuildConfiguration configuration) { + PathFragment parent = configuration.getGenfilesFragment().getParentDirectory(); + return parent.replaceName(parent.getBaseName() + "-lipodata") + .getChild(configuration.getGenfilesFragment().getBaseName()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java new file mode 100644 index 0000000000..ecf3431b4f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java @@ -0,0 +1,1074 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ParameterFile; +import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.extra.CppLinkInfo; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.collect.ImmutableIterable; +import com.google.devtools.build.lib.collect.IterablesChain; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Action that represents an ELF linking step. + */ +@ThreadCompatible +public final class CppLinkAction extends AbstractAction { + private static final String LINK_GUID = "58ec78bd-1176-4e36-8143-439f656b181d"; + private static final String FAKE_LINK_GUID = "da36f819-5a15-43a9-8a45-e01b60e10c8b"; + + private final CppConfiguration cppConfiguration; + private final LibraryToLink outputLibrary; + private final LibraryToLink interfaceOutputLibrary; + + private final LinkCommandLine linkCommandLine; + + /** True for cc_fake_binary targets. */ + private final boolean fake; + + private final Iterable<Artifact> mandatoryInputs; + + // Linking uses a lot of memory; estimate 1 MB per input file, min 1.5 Gib. + // It is vital to not underestimate too much here, + // because running too many concurrent links can + // thrash the machine to the point where it stops + // responding to keystrokes or mouse clicks. + // CPU and IO do not scale similarly and still use the static minimum estimate. + public static final ResourceSet LINK_RESOURCES_PER_INPUT = new ResourceSet(1, 0, 0); + + // This defines the minimum of each resource that will be reserved. + public static final ResourceSet MIN_STATIC_LINK_RESOURCES = new ResourceSet(1536, 1, 0.3); + + // Dynamic linking should be cheaper than static linking. + public static final ResourceSet MIN_DYNAMIC_LINK_RESOURCES = new ResourceSet(1024, 0.3, 0.2); + + /** + * Use {@link Builder} to create instances of this class. Also see there for + * the documentation of all parameters. + * + * <p>This constructor is intentionally private and is only to be called from + * {@link Builder#build()}. + */ + private CppLinkAction(ActionOwner owner, + Iterable<Artifact> inputs, + ImmutableList<Artifact> outputs, + CppConfiguration cppConfiguration, + LibraryToLink outputLibrary, + LibraryToLink interfaceOutputLibrary, + boolean fake, + LinkCommandLine linkCommandLine) { + super(owner, inputs, outputs); + this.mandatoryInputs = inputs; + this.cppConfiguration = cppConfiguration; + this.outputLibrary = outputLibrary; + this.interfaceOutputLibrary = interfaceOutputLibrary; + this.fake = fake; + + this.linkCommandLine = linkCommandLine; + } + + private static Iterable<LinkerInput> filterLinkerInputs(Iterable<LinkerInput> inputs) { + return Iterables.filter(inputs, new Predicate<LinkerInput>() { + @Override + public boolean apply(LinkerInput input) { + return Link.VALID_LINKER_INPUTS.matches(input.getArtifact().getFilename()); + } + }); + } + + private static Iterable<Artifact> filterLinkerInputArtifacts(Iterable<Artifact> inputs) { + return Iterables.filter(inputs, new Predicate<Artifact>() { + @Override + public boolean apply(Artifact input) { + return Link.VALID_LINKER_INPUTS.matches(input.getFilename()); + } + }); + } + + private CppConfiguration getCppConfiguration() { + return cppConfiguration; + } + + @VisibleForTesting + public String getTargetCpu() { + return getCppConfiguration().getTargetCpu(); + } + + public String getHostSystemName() { + return getCppConfiguration().getHostSystemName(); + } + + /** + * Returns the link configuration; for correctness you should not call this method during + * execution - only the argv is part of the action cache key, and we therefore don't guarantee + * that the action will be re-executed if the contents change in a way that does not affect the + * argv. + */ + @VisibleForTesting + public LinkCommandLine getLinkCommandLine() { + return linkCommandLine; + } + + public LibraryToLink getOutputLibrary() { + return outputLibrary; + } + + public LibraryToLink getInterfaceOutputLibrary() { + return interfaceOutputLibrary; + } + + /** + * Returns the path to the output artifact produced by the linker. + */ + public Path getOutputFile() { + return outputLibrary.getArtifact().getPath(); + } + + @VisibleForTesting + public List<String> getRawLinkArgv() { + return linkCommandLine.getRawLinkArgv(); + } + + @VisibleForTesting + public List<String> getArgv() { + return linkCommandLine.arguments(); + } + + /** + * Prepares and returns the command line specification for this link. + * Splits appropriate parts into a .params file and adds any required + * linkstamp compilation steps. + * + * @return a finalized command line suitable for execution + */ + public final List<String> prepareCommandLine(Path execRoot, List<String> inputFiles) + throws ExecException { + List<String> commandlineArgs; + // Try to shorten the command line by use of a parameter file. + // This makes the output with --subcommands (et al) more readable. + if (linkCommandLine.canBeSplit()) { + PathFragment paramExecPath = ParameterFile.derivePath( + outputLibrary.getArtifact().getExecPath()); + Pair<List<String>, List<String>> split = linkCommandLine.splitCommandline(paramExecPath); + commandlineArgs = split.first; + writeToParamFile(execRoot, paramExecPath, split.second); + if (inputFiles != null) { + inputFiles.add(paramExecPath.getPathString()); + } + } else { + commandlineArgs = linkCommandLine.getRawLinkArgv(); + } + return linkCommandLine.finalizeWithLinkstampCommands(commandlineArgs); + } + + private static void writeToParamFile(Path workingDir, PathFragment paramExecPath, + List<String> paramFileArgs) throws ExecException { + // Create parameter file. + ParameterFile paramFile = new ParameterFile(workingDir, paramExecPath, ISO_8859_1, + ParameterFileType.UNQUOTED); + Path paramFilePath = paramFile.getPath(); + try { + // writeContent() fails for existing files that are marked readonly. + paramFilePath.delete(); + } catch (IOException e) { + throw new EnvironmentalExecException("could not delete file '" + paramFilePath + "'", e); + } + paramFile.writeContent(paramFileArgs); + + // Normally Blaze chmods all output files automatically (see + // SkyframeActionExecutor#setOutputsReadOnlyAndExecutable), but this params file is created + // out-of-band and is not declared as an output. By chmodding the file, other processes + // can observe this file being created. + try { + paramFilePath.setWritable(false); + paramFilePath.setExecutable(true); // for consistency with other action outputs + } catch (IOException e) { + throw new EnvironmentalExecException("could not chmod param file '" + paramFilePath + "'", e); + } + } + + @Override + @ThreadCompatible + public void execute( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + if (fake) { + executeFake(); + } else { + Executor executor = actionExecutionContext.getExecutor(); + + try { + executor.getContext(CppLinkActionContext.class).exec( + this, actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException("Linking of rule '" + getOwner().getLabel() + "'", + executor.getVerboseFailures(), this); + } + } + } + + @Override + public String describeStrategy(Executor executor) { + return fake + ? "fake,local" + : executor.getContext(CppLinkActionContext.class).strategyLocality(this); + } + + // Don't forget to update FAKE_LINK_GUID if you modify this method. + @ThreadCompatible + private void executeFake() + throws ActionExecutionException { + // The uses of getLinkConfiguration in this method may not be consistent with the computed key. + // I.e., this may be incrementally incorrect. + final Collection<Artifact> linkstampOutputs = getLinkCommandLine().getLinkstamps().values(); + + // Prefix all fake output files in the command line with $TEST_TMPDIR/. + final String outputPrefix = "$TEST_TMPDIR/"; + List<String> escapedLinkArgv = escapeLinkArgv(linkCommandLine.getRawLinkArgv(), + linkstampOutputs, outputPrefix); + // Write the commands needed to build the real target to the fake target + // file. + StringBuilder s = new StringBuilder(); + Joiner.on('\n').appendTo(s, + "# This is a fake target file, automatically generated.", + "# Do not edit by hand!", + "echo $0 is a fake target file and not meant to be executed.", + "exit 0", + "EOS", + "", + "makefile_dir=.", + ""); + + try { + // Concatenate all the (fake) .o files into the result. + for (LinkerInput linkerInput : getLinkCommandLine().getLinkerInputs()) { + Artifact objectFile = linkerInput.getArtifact(); + if (CppFileTypes.OBJECT_FILE.matches(objectFile.getFilename()) + && linkerInput.isFake()) { + s.append(FileSystemUtils.readContentAsLatin1(objectFile.getPath())); // (IOException) + } + } + + s.append(getOutputFile().getBaseName()).append(": "); + for (Artifact linkstamp : linkstampOutputs) { + s.append("mkdir -p " + outputPrefix + + linkstamp.getExecPath().getParentDirectory() + " && "); + } + Joiner.on(' ').appendTo(s, + ShellEscaper.escapeAll(linkCommandLine.finalizeAlreadyEscapedWithLinkstampCommands( + escapedLinkArgv, outputPrefix))); + s.append('\n'); + if (getOutputFile().exists()) { + getOutputFile().setWritable(true); // (IOException) + } + FileSystemUtils.writeContent(getOutputFile(), ISO_8859_1, s.toString()); + getOutputFile().setExecutable(true); // (IOException) + for (Artifact linkstamp : linkstampOutputs) { + FileSystemUtils.touchFile(linkstamp.getPath()); + } + } catch (IOException e) { + throw new ActionExecutionException("failed to create fake link command for rule '" + + getOwner().getLabel() + ": " + e.getMessage(), + this, false); + } + } + + /** + * Shell-escapes the raw link command line. + * + * @param rawLinkArgv raw link command line + * @param linkstampOutputs linkstamp artifacts + * @param outputPrefix to be prepended to any outputs + * @return escaped link command line + */ + private List<String> escapeLinkArgv(List<String> rawLinkArgv, + final Collection<Artifact> linkstampOutputs, final String outputPrefix) { + final List<String> linkstampExecPaths = Artifact.asExecPaths(linkstampOutputs); + ImmutableList.Builder<String> escapedArgs = ImmutableList.builder(); + for (String rawArg : rawLinkArgv) { + String escapedArg; + if (rawArg.equals(getPrimaryOutput().getExecPathString()) + || linkstampExecPaths.contains(rawArg)) { + escapedArg = outputPrefix + ShellEscaper.escapeString(rawArg); + } else if (rawArg.startsWith(Link.FAKE_OBJECT_PREFIX)) { + escapedArg = outputPrefix + ShellEscaper.escapeString( + rawArg.substring(Link.FAKE_OBJECT_PREFIX.length())); + } else { + escapedArg = ShellEscaper.escapeString(rawArg); + } + escapedArgs.add(escapedArg); + } + return escapedArgs.build(); + } + + @Override + public ExtraActionInfo.Builder getExtraActionInfo() { + // The uses of getLinkConfiguration in this method may not be consistent with the computed key. + // I.e., this may be incrementally incorrect. + CppLinkInfo.Builder info = CppLinkInfo.newBuilder(); + info.addAllInputFile(Artifact.toExecPaths( + LinkerInputs.toLibraryArtifacts(getLinkCommandLine().getLinkerInputs()))); + info.addAllInputFile(Artifact.toExecPaths( + LinkerInputs.toLibraryArtifacts(getLinkCommandLine().getRuntimeInputs()))); + info.setOutputFile(getPrimaryOutput().getExecPathString()); + if (interfaceOutputLibrary != null) { + info.setInterfaceOutputFile(interfaceOutputLibrary.getArtifact().getExecPathString()); + } + info.setLinkTargetType(getLinkCommandLine().getLinkTargetType().name()); + info.setLinkStaticness(getLinkCommandLine().getLinkStaticness().name()); + info.addAllLinkStamp(Artifact.toExecPaths(getLinkCommandLine().getLinkstamps().values())); + info.addAllBuildInfoHeaderArtifact( + Artifact.toExecPaths(getLinkCommandLine().getBuildInfoHeaderArtifacts())); + info.addAllLinkOpt(getLinkCommandLine().getLinkopts()); + + return super.getExtraActionInfo() + .setExtension(CppLinkInfo.cppLinkInfo, info.build()); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(fake ? FAKE_LINK_GUID : LINK_GUID); + f.addString(getCppConfiguration().getLdExecutable().getPathString()); + f.addStrings(linkCommandLine.arguments()); + // TODO(bazel-team): For correctness, we need to ensure the invariant that all values accessed + // during the execution phase are also covered by the key. Above, we add the argv to the key, + // which covers most cases. Unfortunately, the extra action and fake support methods above also + // sometimes directly access settings from the link configuration that may or may not affect the + // key. We either need to change the code to cover them in the key computation, or change the + // LinkConfiguration to disallow the combinations where the value of a setting does not affect + // the argv. + f.addBoolean(linkCommandLine.isNativeDeps()); + f.addBoolean(linkCommandLine.useTestOnlyFlags()); + if (linkCommandLine.getRuntimeSolibDir() != null) { + f.addPath(linkCommandLine.getRuntimeSolibDir()); + } + return f.hexDigestAndReset(); + } + + @Override + public String describeKey() { + StringBuilder message = new StringBuilder(); + if (fake) { + message.append("Fake "); + } + message.append(getProgressMessage()); + message.append('\n'); + message.append(" Command: "); + message.append(ShellEscaper.escapeString( + getCppConfiguration().getLdExecutable().getPathString())); + message.append('\n'); + // Outputting one argument per line makes it easier to diff the results. + for (String argument : ShellEscaper.escapeAll(linkCommandLine.arguments())) { + message.append(" Argument: "); + message.append(argument); + message.append('\n'); + } + return message.toString(); + } + + @Override + public String getMnemonic() { return "CppLink"; } + + @Override + protected String getRawProgressMessage() { + return "Linking " + outputLibrary.getArtifact().prettyPrint(); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return executor.getContext(CppLinkActionContext.class).estimateResourceConsumption(this); + } + + /** + * Estimate the resources consumed when this action is run locally. + */ + public ResourceSet estimateResourceConsumptionLocal() { + // It's ok if this behaves differently even if the key is identical. + ResourceSet minLinkResources = + getLinkCommandLine().getLinkStaticness() == Link.LinkStaticness.DYNAMIC + ? MIN_DYNAMIC_LINK_RESOURCES + : MIN_STATIC_LINK_RESOURCES; + + final int inputSize = Iterables.size(getLinkCommandLine().getLinkerInputs()) + + Iterables.size(getLinkCommandLine().getRuntimeInputs()); + + return new ResourceSet( + Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getMemoryMb(), + minLinkResources.getMemoryMb()), + Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getCpuUsage(), + minLinkResources.getCpuUsage()), + Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getIoUsage(), + minLinkResources.getIoUsage()) + ); + } + + @Override + public Iterable<Artifact> getMandatoryInputs() { + return mandatoryInputs; + } + + /** + * Determines whether or not this link should output a symbol counts file. + */ + private static boolean enableSymbolsCounts(CppConfiguration cppConfiguration, boolean fake, + LinkTargetType linkType) { + return cppConfiguration.getSymbolCounts() + && cppConfiguration.supportsGoldLinker() + && linkType == LinkTargetType.EXECUTABLE + && !fake; + } + + /** + * Builder class to construct {@link CppLinkAction}s. + */ + public static class Builder { + // Builder-only + private final RuleContext ruleContext; + private final AnalysisEnvironment analysisEnvironment; + private final PathFragment outputPath; + private final CcToolchainProvider toolchain; + private PathFragment interfaceOutputPath; + private PathFragment runtimeSolibDir; + protected final BuildConfiguration configuration; + private final CppConfiguration cppConfiguration; + + // Morally equivalent with {@link Context}, except these are mutable. + // Keep these in sync with {@link Context}. + private final Set<LinkerInput> nonLibraries = new LinkedHashSet<>(); + private final NestedSetBuilder<LibraryToLink> libraries = NestedSetBuilder.linkOrder(); + private NestedSet<Artifact> crosstoolInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + private Artifact runtimeMiddleman; + private NestedSet<Artifact> runtimeInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + private final NestedSetBuilder<Artifact> compilationInputs = NestedSetBuilder.stableOrder(); + private final Set<Artifact> linkstamps = new LinkedHashSet<>(); + private List<String> linkstampOptions = new ArrayList<>(); + private final List<String> linkopts = new ArrayList<>(); + private LinkTargetType linkType = LinkTargetType.STATIC_LIBRARY; + private LinkStaticness linkStaticness = LinkStaticness.FULLY_STATIC; + private boolean fake; + private boolean isNativeDeps; + private boolean useTestOnlyFlags; + private boolean wholeArchive; + private boolean supportsParamFiles = true; + + /** + * Creates a builder that builds {@link CppLinkAction} instances. + * + * @param ruleContext the rule that owns the action + * @param outputPath the path of the ELF file to be created, relative to the + * 'bin' directory + */ + public Builder(RuleContext ruleContext, PathFragment outputPath) { + this(ruleContext, outputPath, ruleContext.getConfiguration(), + ruleContext.getAnalysisEnvironment(), CppHelper.getToolchain(ruleContext)); + } + + /** + * Creates a builder that builds {@link CppLinkAction} instances. + * + * @param ruleContext the rule that owns the action + * @param outputPath the path of the ELF file to be created, relative to the + * 'bin' directory + */ + public Builder(RuleContext ruleContext, PathFragment outputPath, + BuildConfiguration configuration, CcToolchainProvider toolchain) { + this(ruleContext, outputPath, configuration, + ruleContext.getAnalysisEnvironment(), toolchain); + } + + /** + * Creates a builder that builds {@link CppLinkAction}s. + * + * @param ruleContext the rule that owns the action + * @param outputPath the path of the ELF file to be created, relative to the + * 'bin' directory + * @param configuration the configuration used to determine the tool chain + * and the default link options + */ + private Builder(RuleContext ruleContext, PathFragment outputPath, + BuildConfiguration configuration, AnalysisEnvironment analysisEnvironment, + CcToolchainProvider toolchain) { + this.ruleContext = ruleContext; + this.analysisEnvironment = Preconditions.checkNotNull(analysisEnvironment); + this.outputPath = Preconditions.checkNotNull(outputPath); + this.configuration = Preconditions.checkNotNull(configuration); + this.cppConfiguration = configuration.getFragment(CppConfiguration.class); + this.toolchain = toolchain; + + // The toolchain != null is here for CppLinkAction.createTestBuilder(). Meh. + if (cppConfiguration.supportsEmbeddedRuntimes() && toolchain != null) { + runtimeSolibDir = toolchain.getDynamicRuntimeSolibDir(); + } + if (toolchain != null) { + supportsParamFiles = toolchain.supportsParamFiles(); + } + } + + /** + * Given a Context, creates a Builder that builds {@link CppLinkAction}s. + * Note well: Keep the Builder->Context and Context->Builder transforms consistent! + * @param ruleContext the rule that owns the action + * @param outputPath the path of the ELF file to be created, relative to the + * 'bin' directory + * @param linkContext an immutable CppLinkAction.Context from the original builder + */ + public Builder(RuleContext ruleContext, PathFragment outputPath, Context linkContext, + BuildConfiguration configuration) { + // These Builder-only fields get set in the constructor: + // ruleContext, analysisEnvironment, outputPath, configuration, runtimeSolibDir + this(ruleContext, outputPath, configuration, ruleContext.getAnalysisEnvironment(), + CppHelper.getToolchain(ruleContext)); + Preconditions.checkNotNull(linkContext); + + // All linkContext fields should be transferred to this Builder. + this.nonLibraries.addAll(linkContext.nonLibraries); + this.libraries.addTransitive(linkContext.libraries); + this.crosstoolInputs = linkContext.crosstoolInputs; + this.runtimeMiddleman = linkContext.runtimeMiddleman; + this.runtimeInputs = linkContext.runtimeInputs; + this.compilationInputs.addTransitive(linkContext.compilationInputs); + this.linkstamps.addAll(linkContext.linkstamps); + this.linkopts.addAll(linkContext.linkopts); + this.linkType = linkContext.linkType; + this.linkStaticness = linkContext.linkStaticness; + this.fake = linkContext.fake; + this.isNativeDeps = linkContext.isNativeDeps; + this.useTestOnlyFlags = linkContext.useTestOnlyFlags; + } + + /** + * Builds the Action as configured and returns it. + * + * <p>This method may only be called once. + */ + public CppLinkAction build() { + if (interfaceOutputPath != null && (fake || linkType != LinkTargetType.DYNAMIC_LIBRARY)) { + throw new RuntimeException("Interface output can only be used " + + "with non-fake DYNAMIC_LIBRARY targets"); + } + + final Artifact output = createArtifact(outputPath); + final Artifact interfaceOutput = (interfaceOutputPath != null) + ? createArtifact(interfaceOutputPath) + : null; + + final ImmutableList<Artifact> buildInfoHeaderArtifacts = !linkstamps.isEmpty() + ? ruleContext.getAnalysisEnvironment().getBuildInfo(ruleContext, CppBuildInfo.KEY) + : ImmutableList.<Artifact>of(); + + final Artifact symbolCountOutput = enableSymbolsCounts(cppConfiguration, fake, linkType) + ? createArtifact(output.getRootRelativePath().replaceName( + output.getExecPath().getBaseName() + ".sc")) + : null; + + boolean needWholeArchive = wholeArchive || needWholeArchive( + linkStaticness, linkType, linkopts, isNativeDeps, cppConfiguration); + + NestedSet<LibraryToLink> uniqueLibraries = libraries.build(); + final Iterable<Artifact> filteredNonLibraryArtifacts = filterLinkerInputArtifacts( + LinkerInputs.toLibraryArtifacts(nonLibraries)); + final Iterable<LinkerInput> linkerInputs = IterablesChain.<LinkerInput>builder() + .add(ImmutableList.copyOf(filterLinkerInputs(nonLibraries))) + .add(ImmutableIterable.from(Link.mergeInputsCmdLine( + uniqueLibraries, needWholeArchive, cppConfiguration.archiveType()))) + .build(); + + // ruleContext can only be null during testing. This is kind of ugly. + final ImmutableSet<String> features = (ruleContext == null) + ? ImmutableSet.<String>of() + : ruleContext.getFeatures(); + + final LibraryToLink outputLibrary = + LinkerInputs.newInputLibrary(output, filteredNonLibraryArtifacts); + final LibraryToLink interfaceOutputLibrary = interfaceOutput == null ? null : + LinkerInputs.newInputLibrary(interfaceOutput, filteredNonLibraryArtifacts); + + final ImmutableMap<Artifact, Artifact> linkstampMap = + mapLinkstampsToOutputs(linkstamps, ruleContext, output); + + final ImmutableList<Artifact> actionOutputs = constructOutputs( + outputLibrary.getArtifact(), + linkstampMap.values(), + interfaceOutputLibrary == null ? null : interfaceOutputLibrary.getArtifact(), + symbolCountOutput); + + LinkCommandLine linkCommandLine = new LinkCommandLine.Builder(configuration, getOwner()) + .setOutput(outputLibrary.getArtifact()) + .setInterfaceOutput(interfaceOutput) + .setSymbolCountsOutput(symbolCountOutput) + .setBuildInfoHeaderArtifacts(buildInfoHeaderArtifacts) + .setLinkerInputs(linkerInputs) + .setRuntimeInputs(ImmutableList.copyOf(LinkerInputs.simpleLinkerInputs(runtimeInputs))) + .setLinkTargetType(linkType) + .setLinkStaticness(linkStaticness) + .setLinkopts(ImmutableList.copyOf(linkopts)) + .setFeatures(features) + .setLinkstamps(linkstampMap) + .addLinkstampCompileOptions(linkstampOptions) + .setRuntimeSolibDir(linkType.isStaticLibraryLink() ? null : runtimeSolibDir) + .setNativeDeps(isNativeDeps) + .setUseTestOnlyFlags(useTestOnlyFlags) + .setNeedWholeArchive(needWholeArchive) + .setInterfaceSoBuilder(getInterfaceSoBuilder()) + .setSupportsParamFiles(supportsParamFiles) + .build(); + + // Compute the set of inputs - we only need stable order here. + NestedSetBuilder<Artifact> dependencyInputsBuilder = NestedSetBuilder.stableOrder(); + dependencyInputsBuilder.addAll(buildInfoHeaderArtifacts); + dependencyInputsBuilder.addAll(linkstamps); + dependencyInputsBuilder.addTransitive(crosstoolInputs); + if (runtimeMiddleman != null) { + dependencyInputsBuilder.add(runtimeMiddleman); + } + dependencyInputsBuilder.addTransitive(compilationInputs.build()); + + Iterable<Artifact> expandedInputs = + LinkerInputs.toLibraryArtifacts(Link.mergeInputsDependencies(uniqueLibraries, + needWholeArchive, cppConfiguration.archiveType())); + // getPrimaryInput returns the first element, and that is a public interface - therefore the + // order here is important. + Iterable<Artifact> inputs = IterablesChain.<Artifact>builder() + .add(ImmutableList.copyOf(LinkerInputs.toLibraryArtifacts(nonLibraries))) + .add(dependencyInputsBuilder.build()) + .add(ImmutableIterable.from(expandedInputs)) + .deduplicate() + .build(); + + return new CppLinkAction( + getOwner(), + inputs, + actionOutputs, + cppConfiguration, + outputLibrary, + interfaceOutputLibrary, + fake, + linkCommandLine); + } + + /** + * The default heuristic on whether we need to use whole-archive for the link. + */ + private static boolean needWholeArchive(LinkStaticness staticness, + LinkTargetType type, Collection<String> linkopts, boolean isNativeDeps, + CppConfiguration cppConfig) { + boolean fullyStatic = (staticness == LinkStaticness.FULLY_STATIC); + boolean mostlyStatic = (staticness == LinkStaticness.MOSTLY_STATIC); + boolean sharedLinkopts = type == LinkTargetType.DYNAMIC_LIBRARY + || linkopts.contains("-shared") + || cppConfig.getLinkOptions().contains("-shared"); + return (isNativeDeps || cppConfig.legacyWholeArchive()) + && (fullyStatic || mostlyStatic) + && sharedLinkopts; + } + + private static ImmutableList<Artifact> constructOutputs(Artifact primaryOutput, + Collection<Artifact> outputList, Artifact... outputs) { + return new ImmutableList.Builder<Artifact>() + .add(primaryOutput) + .addAll(outputList) + .addAll(CollectionUtils.asListWithoutNulls(outputs)) + .build(); + } + + /** + * Translates a collection of linkstamp source files to an immutable + * mapping from source files to object files. In other words, given a + * set of source files, this method determines the output path to which + * each file should be compiled. + * + * @param linkstamps collection of linkstamp source files + * @param ruleContext the rule for which this link is being performed + * @param outputBinary the binary output path for this link + * @return an immutable map that pairs each source file with the + * corresponding object file that should be fed into the link + */ + public static ImmutableMap<Artifact, Artifact> mapLinkstampsToOutputs( + Collection<Artifact> linkstamps, RuleContext ruleContext, Artifact outputBinary) { + ImmutableMap.Builder<Artifact, Artifact> mapBuilder = ImmutableMap.builder(); + + PathFragment outputBinaryPath = outputBinary.getRootRelativePath(); + PathFragment stampOutputDirectory = outputBinaryPath.getParentDirectory(). + getRelative("_objs").getRelative(outputBinaryPath.getBaseName()); + + for (Artifact linkstamp : linkstamps) { + PathFragment stampOutputPath = stampOutputDirectory.getRelative( + FileSystemUtils.replaceExtension(linkstamp.getRootRelativePath(), ".o")); + mapBuilder.put(linkstamp, + ruleContext.getAnalysisEnvironment().getDerivedArtifact( + stampOutputPath, outputBinary.getRoot())); + } + return mapBuilder.build(); + } + + protected ActionOwner getOwner() { + return ruleContext.getActionOwner(); + } + + protected Artifact createArtifact(PathFragment path) { + return analysisEnvironment.getDerivedArtifact(path, configuration.getBinDirectory()); + } + + protected Artifact getInterfaceSoBuilder() { + return analysisEnvironment.getEmbeddedToolArtifact(CppRuleClasses.BUILD_INTERFACE_SO); + } + + /** + * Set the crosstool inputs required for the action. + */ + public Builder setCrosstoolInputs(NestedSet<Artifact> inputs) { + this.crosstoolInputs = inputs; + return this; + } + + /** + * Sets the C++ runtime library inputs for the action. + */ + public Builder setRuntimeInputs(Artifact middleman, NestedSet<Artifact> inputs) { + Preconditions.checkArgument((middleman == null) == inputs.isEmpty()); + this.runtimeMiddleman = middleman; + this.runtimeInputs = inputs; + return this; + } + + /** + * Sets the interface output of the link. A non-null argument can + * only be provided if the link type is {@code DYNAMIC_LIBRARY} + * and fake is false. + */ + public Builder setInterfaceOutputPath(PathFragment path) { + this.interfaceOutputPath = path; + return this; + } + + /** + * Add additional inputs needed for the linkstamp compilation that is being done as part of the + * link. + */ + public Builder addCompilationInputs(Iterable<Artifact> inputs) { + this.compilationInputs.addAll(inputs); + return this; + } + + public Builder addTransitiveCompilationInputs(NestedSet<Artifact> inputs) { + this.compilationInputs.addTransitive(inputs); + return this; + } + + private void addNonLibraryInput(LinkerInput input) { + String name = input.getArtifact().getFilename(); + Preconditions.checkArgument( + !Link.ARCHIVE_LIBRARY_FILETYPES.matches(name) + && !Link.SHARED_LIBRARY_FILETYPES.matches(name), + "'%s' is a library file", input); + this.nonLibraries.add(input); + } + /** + * Adds a single artifact to the set of inputs (C++ source files, header files, etc). Artifacts + * that are not of recognized types will be used for dependency checking but will not be passed + * to the linker. The artifact must not be an archive or a shared library. + */ + public Builder addNonLibraryInput(Artifact input) { + addNonLibraryInput(LinkerInputs.simpleLinkerInput(input)); + return this; + } + + /** + * Adds multiple artifacts to the set of inputs (C++ source files, header files, etc). + * Artifacts that are not of recognized types will be used for dependency checking but will + * not be passed to the linker. The artifacts must not be archives or shared libraries. + */ + public Builder addNonLibraryInputs(Iterable<Artifact> inputs) { + for (Artifact input : inputs) { + addNonLibraryInput(LinkerInputs.simpleLinkerInput(input)); + } + return this; + } + + public Builder addFakeNonLibraryInputs(Iterable<Artifact> inputs) { + for (Artifact input : inputs) { + addNonLibraryInput(LinkerInputs.fakeLinkerInput(input)); + } + return this; + } + + private void checkLibrary(LibraryToLink input) { + String name = input.getArtifact().getFilename(); + Preconditions.checkArgument( + Link.ARCHIVE_LIBRARY_FILETYPES.matches(name) || + Link.SHARED_LIBRARY_FILETYPES.matches(name), + "'%s' is not a library file", input); + } + + /** + * Adds a single artifact to the set of inputs. The artifact must be an archive or a shared + * library. Note that all directly added libraries are implicitly ordered before all nested + * sets added with {@link #addLibraries}, even if added in the opposite order. + */ + public Builder addLibrary(LibraryToLink input) { + checkLibrary(input); + libraries.add(input); + return this; + } + + /** + * Adds multiple artifact to the set of inputs. The artifacts must be archives or shared + * libraries. + */ + public Builder addLibraries(NestedSet<LibraryToLink> inputs) { + for (LibraryToLink input : inputs) { + checkLibrary(input); + } + this.libraries.addTransitive(inputs); + return this; + } + + /** + * Sets the type of ELF file to be created (.a, .so, .lo, executable). The + * default is {@link LinkTargetType#STATIC_LIBRARY}. + */ + public Builder setLinkType(LinkTargetType linkType) { + this.linkType = linkType; + return this; + } + + /** + * Sets the degree of "staticness" of the link: fully static (static binding + * of all symbols), mostly static (use dynamic binding only for symbols from + * glibc), dynamic (use dynamic binding wherever possible). The default is + * {@link LinkStaticness#FULLY_STATIC}. + */ + public Builder setLinkStaticness(LinkStaticness linkStaticness) { + this.linkStaticness = linkStaticness; + return this; + } + + /** + * Adds a C++ source file which will be compiled at link time. This is used + * to embed various values from the build system into binaries to identify + * their provenance. + * + * <p>Link stamps are also automatically added to the inputs. + */ + public Builder addLinkstamps(Map<Artifact, ImmutableList<Artifact>> linkstamps) { + this.linkstamps.addAll(linkstamps.keySet()); + // Add inputs for linkstamping. + if (!linkstamps.isEmpty()) { + // This will just be the compiler unless include scanning is disabled, in which case it will + // include all header files. Since we insist that linkstamps declare all their headers, all + // header files would be overkill, but that only happens when include scanning is disabled. + addTransitiveCompilationInputs(toolchain.getCompile()); + for (Map.Entry<Artifact, ImmutableList<Artifact>> entry : linkstamps.entrySet()) { + addCompilationInputs(entry.getValue()); + } + } + return this; + } + + public Builder addLinkstampCompilerOptions(ImmutableList<String> linkstampOptions) { + this.linkstampOptions = linkstampOptions; + return this; + } + + /** + * Adds an additional linker option. + */ + public Builder addLinkopt(String linkopt) { + this.linkopts.add(linkopt); + return this; + } + + /** + * Adds multiple linker options at once. + * + * @see #addLinkopt(String) + */ + public Builder addLinkopts(Collection<String> linkopts) { + this.linkopts.addAll(linkopts); + return this; + } + + /** + * Sets whether this link action will be used for a cc_fake_binary; false by + * default. + */ + public Builder setFake(boolean fake) { + this.fake = fake; + return this; + } + + /** + * Sets whether this link action is used for a native dependency library. + */ + public Builder setNativeDeps(boolean isNativeDeps) { + this.isNativeDeps = isNativeDeps; + return this; + } + + /** + * Setting this to true overrides the default whole-archive computation and force-enables + * whole archives for every archive in the link. This is only necessary for linking executable + * binaries that are supposed to export symbols. + * + * <p>Usually, the link action while use whole archives for dynamic libraries that are native + * deps (or the legacy whole archive flag is enabled), and that are not dynamically linked. + * + * <p>(Note that it is possible to build dynamic libraries with cc_binary rules by specifying + * linkshared = 1, and giving the rule a name that matches the pattern {@code + * lib<name>.so}.) + */ + public Builder setWholeArchive(boolean wholeArchive) { + this.wholeArchive = wholeArchive; + return this; + } + + /** + * Sets whether this link action should use test-specific flags (e.g. $EXEC_ORIGIN instead of + * $ORIGIN for the solib search path or lazy binding); false by default. + */ + public Builder setUseTestOnlyFlags(boolean useTestOnlyFlags) { + this.useTestOnlyFlags = useTestOnlyFlags; + return this; + } + + /** + * Sets the name of the directory where the solib symlinks for the dynamic runtime libraries + * live. This is usually automatically set from the cc_toolchain. + */ + public Builder setRuntimeSolibDir(PathFragment runtimeSolibDir) { + this.runtimeSolibDir = runtimeSolibDir; + return this; + } + + /** + * Creates a builder without the need for a {@link RuleContext}. + * This is to be used exclusively for testing purposes. + * + * <p>Link stamping is not supported if using this method. + */ + @VisibleForTesting + public static Builder createTestBuilder( + final ActionOwner owner, final AnalysisEnvironment analysisEnvironment, + final PathFragment outputPath, BuildConfiguration config) { + return new Builder(null, outputPath, config, analysisEnvironment, null) { + @Override + protected Artifact createArtifact(PathFragment path) { + return new Artifact(configuration.getBinDirectory().getPath().getRelative(path), + configuration.getBinDirectory(), configuration.getBinFragment().getRelative(path), + analysisEnvironment.getOwner()); + } + @Override + protected ActionOwner getOwner() { + return owner; + } + }; + } + } + + /** + * Immutable ELF linker context, suitable for serialization. + */ + @Immutable @ThreadSafe + public static final class Context implements TransitiveInfoProvider { + // Morally equivalent with {@link Builder}, except these are immutable. + // Keep these in sync with {@link Builder}. + private final ImmutableSet<LinkerInput> nonLibraries; + private final NestedSet<LibraryToLink> libraries; + private final NestedSet<Artifact> crosstoolInputs; + private final Artifact runtimeMiddleman; + private final NestedSet<Artifact> runtimeInputs; + private final NestedSet<Artifact> compilationInputs; + private final ImmutableSet<Artifact> linkstamps; + private final ImmutableList<String> linkopts; + private final LinkTargetType linkType; + private final LinkStaticness linkStaticness; + private final boolean fake; + private final boolean isNativeDeps; + private final boolean useTestOnlyFlags; + + /** + * Given a {@link Builder}, creates a {@code Context} to pass to another target. + * Note well: Keep the Builder->Context and Context->Builder transforms consistent! + * @param builder a mutable {@link CppLinkAction.Builder} to clone from + */ + public Context(Builder builder) { + this.nonLibraries = ImmutableSet.copyOf(builder.nonLibraries); + this.libraries = NestedSetBuilder.<LibraryToLink>linkOrder() + .addTransitive(builder.libraries.build()).build(); + this.crosstoolInputs = + NestedSetBuilder.<Artifact>stableOrder().addTransitive(builder.crosstoolInputs).build(); + this.runtimeMiddleman = builder.runtimeMiddleman; + this.runtimeInputs = + NestedSetBuilder.<Artifact>stableOrder().addTransitive(builder.runtimeInputs).build(); + this.compilationInputs = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(builder.compilationInputs.build()).build(); + this.linkstamps = ImmutableSet.copyOf(builder.linkstamps); + this.linkopts = ImmutableList.copyOf(builder.linkopts); + this.linkType = builder.linkType; + this.linkStaticness = builder.linkStaticness; + this.fake = builder.fake; + this.isNativeDeps = builder.isNativeDeps; + this.useTestOnlyFlags = builder.useTestOnlyFlags; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionContext.java new file mode 100644 index 0000000000..24a936b281 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionContext.java @@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.ActionContextMarker; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.ResourceSet; + +/** + * Context for executing {@link CppLinkAction}s. + */ +@ActionContextMarker(name = "C++ link") +public interface CppLinkActionContext extends ActionContext { + /** + * Returns where the action actually runs. + */ + String strategyLocality(CppLinkAction action); + + /** + * Returns the estimated resource consumption of the action. + */ + ResourceSet estimateResourceConsumption(CppLinkAction action); + + /** + * Executes the specified action. + */ + void exec(CppLinkAction action, + ActionExecutionContext actionExecutionContext) + throws ExecException, ActionExecutionException, InterruptedException; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModel.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModel.java new file mode 100644 index 0000000000..44258a5ab1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModel.java @@ -0,0 +1,707 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CcCompilationOutputs.Builder; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.RegexFilter; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +/** + * Representation of a C/C++ compilation. Its purpose is to share the code that creates compilation + * actions between all classes that need to do so. It follows the builder pattern - load up the + * necessary settings and then call {@link #createCcCompileActions}. + * + * <p>This class is not thread-safe, and it should only be used once for each set of source files, + * i.e. calling {@link #createCcCompileActions} will throw an Exception if called twice. + */ +public final class CppModel { + private final CppSemantics semantics; + private final RuleContext ruleContext; + private final BuildConfiguration configuration; + private final CppConfiguration cppConfiguration; + + // compile model + private CppCompilationContext context; + private final List<Pair<Artifact, Label>> sourceFiles = new ArrayList<>(); + private final List<String> copts = new ArrayList<>(); + private final List<PathFragment> additionalIncludes = new ArrayList<>(); + @Nullable private Pattern nocopts; + private boolean fake; + private boolean maySaveTemps; + private boolean onlySingleOutput; + private CcCompilationOutputs compilationOutputs; + private boolean enableLayeringCheck; + private boolean compileHeaderModules; + + // link model + private final List<String> linkopts = new ArrayList<>(); + private LinkTargetType linkType = LinkTargetType.STATIC_LIBRARY; + private boolean neverLink; + private boolean allowInterfaceSharedObjects; + private boolean createDynamicLibrary = true; + private PathFragment soImplFilename; + private FeatureConfiguration featureConfiguration; + + public CppModel(RuleContext ruleContext, CppSemantics semantics) { + this.ruleContext = ruleContext; + this.semantics = semantics; + configuration = ruleContext.getConfiguration(); + cppConfiguration = configuration.getFragment(CppConfiguration.class); + } + + /** + * If the cpp compilation is a fake, then it creates only a single compile action without PIC. + * Defaults to false. + */ + public CppModel setFake(boolean fake) { + this.fake = fake; + return this; + } + + /** + * If set, the CppModel only creates a single .o output that can be linked into a dynamic library, + * i.e., it never generates both PIC and non-PIC outputs. Otherwise it creates outputs that can be + * linked into both static binaries and dynamic libraries (if both require PIC or both require + * non-PIC, then it still only creates a single output). Defaults to false. + */ + public CppModel setOnlySingleOutput(boolean onlySingleOutput) { + this.onlySingleOutput = onlySingleOutput; + return this; + } + + /** + * If set, use compiler flags to enable compiler based layering checks. + */ + public CppModel setEnableLayeringCheck(boolean enableLayeringCheck) { + this.enableLayeringCheck = enableLayeringCheck; + return this; + } + + /** + * If set, add actions that compile header modules to the build. + * See http://clang.llvm.org/docs/Modules.html for more information. + */ + public CppModel setCompileHeaderModules(boolean compileHeaderModules) { + this.compileHeaderModules = compileHeaderModules; + return this; + } + + /** + * Whether to create actions for temps. This defaults to false. + */ + public CppModel setSaveTemps(boolean maySaveTemps) { + this.maySaveTemps = maySaveTemps; + return this; + } + + /** + * Sets the compilation context, i.e. include directories and allowed header files inclusions. + */ + public CppModel setContext(CppCompilationContext context) { + this.context = context; + return this; + } + + /** + * Adds a single source file to be compiled. Note that this should only be called for primary + * compilation units, not for header files or files that are otherwise included. + */ + public CppModel addSources(Iterable<Artifact> sourceFiles, Label sourceLabel) { + for (Artifact sourceFile : sourceFiles) { + this.sourceFiles.add(Pair.of(sourceFile, sourceLabel)); + } + return this; + } + + /** + * Adds all the source files. Note that this should only be called for primary compilation units, + * not for header files or files that are otherwise included. + */ + public CppModel addSources(Iterable<Pair<Artifact, Label>> sources) { + Iterables.addAll(this.sourceFiles, sources); + return this; + } + + /** + * Adds the given copts. + */ + public CppModel addCopts(Collection<String> copts) { + this.copts.addAll(copts); + return this; + } + + /** + * Sets the nocopts pattern. This is used to filter out flags from the system defined set of + * flags. By default no filter is applied. + */ + public CppModel setNoCopts(@Nullable Pattern nocopts) { + this.nocopts = nocopts; + return this; + } + + /** + * This can be used to specify additional include directories, without modifying the compilation + * context. + */ + public CppModel addAdditionalIncludes(Collection<PathFragment> additionalIncludes) { + // TODO(bazel-team): Maybe this could be handled by the compilation context instead? + this.additionalIncludes.addAll(additionalIncludes); + return this; + } + + /** + * Adds the given linkopts to the optional dynamic library link command. + */ + public CppModel addLinkopts(Collection<String> linkopts) { + this.linkopts.addAll(linkopts); + return this; + } + + /** + * Sets the link type used for the link actions. Note that only static links are supported at this + * time. + */ + public CppModel setLinkTargetType(LinkTargetType linkType) { + this.linkType = linkType; + return this; + } + + public CppModel setNeverLink(boolean neverLink) { + this.neverLink = neverLink; + return this; + } + + /** + * Whether to allow interface dynamic libraries. Note that setting this to true only has an effect + * if the configuration allows it. Defaults to false. + */ + public CppModel setAllowInterfaceSharedObjects(boolean allowInterfaceSharedObjects) { + // TODO(bazel-team): Set the default to true, and require explicit action to disable it. + this.allowInterfaceSharedObjects = allowInterfaceSharedObjects; + return this; + } + + public CppModel setCreateDynamicLibrary(boolean createDynamicLibrary) { + this.createDynamicLibrary = createDynamicLibrary; + return this; + } + + public CppModel setDynamicLibraryPath(PathFragment soImplFilename) { + this.soImplFilename = soImplFilename; + return this; + } + + /** + * Sets the feature configuration to be used for C/C++ actions. + */ + public CppModel setFeatureConfiguration(FeatureConfiguration featureConfiguration) { + this.featureConfiguration = featureConfiguration; + return this; + } + + /** + * @return the non-pic header module artifact for the current target. + */ + public Artifact getHeaderModule(Artifact moduleMapArtifact) { + PathFragment objectDir = CppHelper.getObjDirectory(ruleContext.getLabel()); + PathFragment outputName = objectDir.getRelative( + semantics.getEffectiveSourcePath(moduleMapArtifact)); + return ruleContext.getRelatedArtifact(outputName, ".pcm"); + } + + /** + * @return the pic header module artifact for the current target. + */ + public Artifact getPicHeaderModule(Artifact moduleMapArtifact) { + PathFragment objectDir = CppHelper.getObjDirectory(ruleContext.getLabel()); + PathFragment outputName = objectDir.getRelative( + semantics.getEffectiveSourcePath(moduleMapArtifact)); + return ruleContext.getRelatedArtifact(outputName, ".pic.pcm"); + } + + /** + * @return whether this target needs to generate pic actions. + */ + public boolean getGeneratePicActions() { + return CppHelper.usePic(ruleContext, false); + } + + /** + * @return whether this target needs to generate non-pic actions. + */ + public boolean getGenerateNoPicActions() { + return + // If we always need pic for everything, then don't bother to create a no-pic action. + (!CppHelper.usePic(ruleContext, true) || !CppHelper.usePic(ruleContext, false)) + // onlySingleOutput guarantees that the code is only ever linked into a dynamic library - so + // we don't need a no-pic action even if linking into a binary would require it. + && !((onlySingleOutput && getGeneratePicActions())); + } + + /** + * @return whether this target needs to generate a pic header module. + */ + public boolean getGeneratesPicHeaderModule() { + // TODO(bazel-team): Make sure cc_fake_binary works with header module support. + return compileHeaderModules && !fake && getGeneratePicActions(); + } + + /** + * @return whether this target needs to generate a non-pic header module. + */ + public boolean getGeratesNoPicHeaderModule() { + return compileHeaderModules && !fake && getGenerateNoPicActions(); + } + + /** + * Returns a {@code CppCompileActionBuilder} with the common fields for a C++ compile action + * being initialized. + */ + private CppCompileActionBuilder initializeCompileAction(Artifact sourceArtifact, + Label sourceLabel) { + CppCompileActionBuilder builder = createCompileActionBuilder(sourceArtifact, sourceLabel); + if (nocopts != null) { + builder.addNocopts(nocopts); + } + + builder.setEnableLayeringCheck(enableLayeringCheck); + builder.setCompileHeaderModules(compileHeaderModules); + builder.setExtraSystemIncludePrefixes(additionalIncludes); + builder.setFdoBuildStamp(CppHelper.getFdoBuildStamp(cppConfiguration)); + builder.setFeatureConfiguration(featureConfiguration); + return builder; + } + + /** + * Constructs the C++ compiler actions. It generally creates one action for every specified source + * file. It takes into account LIPO, fake-ness, coverage, and PIC, in addition to using the + * settings specified on the current object. This method should only be called once. + */ + public CcCompilationOutputs createCcCompileActions() { + CcCompilationOutputs.Builder result = new CcCompilationOutputs.Builder(); + Preconditions.checkNotNull(context); + AnalysisEnvironment env = ruleContext.getAnalysisEnvironment(); + PathFragment objectDir = CppHelper.getObjDirectory(ruleContext.getLabel()); + + if (compileHeaderModules) { + Artifact moduleMapArtifact = context.getCppModuleMap().getArtifact(); + Label moduleMapLabel = Label.parseAbsoluteUnchecked(context.getCppModuleMap().getName()); + PathFragment outputName = getObjectOutputPath(moduleMapArtifact, objectDir); + CppCompileActionBuilder builder = initializeCompileAction(moduleMapArtifact, moduleMapLabel); + + // A header module compile action is just like a normal compile action, but: + // - the compiled source file is the module map + // - it creates a header module (.pcm file). + createSourceAction(outputName, result, env, moduleMapArtifact, builder, ".pcm"); + } + + for (Pair<Artifact, Label> source : sourceFiles) { + Artifact sourceArtifact = source.getFirst(); + Label sourceLabel = source.getSecond(); + PathFragment outputName = getObjectOutputPath(sourceArtifact, objectDir); + CppCompileActionBuilder builder = initializeCompileAction(sourceArtifact, sourceLabel); + + if (CppFileTypes.CPP_HEADER.matches(source.first.getExecPath())) { + createHeaderAction(outputName, result, env, builder); + } else { + createSourceAction(outputName, result, env, sourceArtifact, builder, ".o"); + } + } + + compilationOutputs = result.build(); + return compilationOutputs; + } + + private void createHeaderAction(PathFragment outputName, Builder result, AnalysisEnvironment env, + CppCompileActionBuilder builder) { + builder.setOutputFile(ruleContext.getRelatedArtifact(outputName, ".h.processed")).setDotdFile( + outputName, ".h.d", ruleContext); + semantics.finalizeCompileActionBuilder(ruleContext, builder); + CppCompileAction compileAction = builder.build(); + env.registerAction(compileAction); + Artifact tokenFile = compileAction.getOutputFile(); + result.addHeaderTokenFile(tokenFile); + } + + private void createSourceAction(PathFragment outputName, + CcCompilationOutputs.Builder result, + AnalysisEnvironment env, + Artifact sourceArtifact, + CppCompileActionBuilder builder, + String outputExtension) { + PathFragment ccRelativeName = semantics.getEffectiveSourcePath(sourceArtifact); + LipoContextProvider lipoProvider = null; + if (cppConfiguration.isLipoOptimization()) { + // TODO(bazel-team): we shouldn't be needing this, merging context with the binary + // is a superset of necessary information. + lipoProvider = Preconditions.checkNotNull(CppHelper.getLipoContextProvider(ruleContext), + outputName); + builder.setContext(CppCompilationContext.mergeForLipo(lipoProvider.getLipoContext(), + context)); + } + if (fake) { + // For cc_fake_binary, we only create a single fake compile action. It's + // not necessary to use -fPIC for negative compilation tests, and using + // .pic.o files in cc_fake_binary would break existing uses of + // cc_fake_binary. + Artifact outputFile = ruleContext.getRelatedArtifact(outputName, outputExtension); + PathFragment tempOutputName = + FileSystemUtils.replaceExtension(outputFile.getExecPath(), ".temp" + outputExtension); + builder + .setOutputFile(outputFile) + .setDotdFile(outputName, ".d", ruleContext) + .setTempOutputFile(tempOutputName); + semantics.finalizeCompileActionBuilder(ruleContext, builder); + CppCompileAction action = builder.build(); + env.registerAction(action); + result.addObjectFile(action.getOutputFile()); + } else { + boolean generatePicAction = getGeneratePicActions(); + // If we always need pic for everything, then don't bother to create a no-pic action. + boolean generateNoPicAction = getGenerateNoPicActions(); + Preconditions.checkState(generatePicAction || generateNoPicAction); + + // Create PIC compile actions (same as non-PIC, but use -fPIC and + // generate .pic.o, .pic.d, .pic.gcno instead of .o, .d, .gcno.) + if (generatePicAction) { + CppCompileActionBuilder picBuilder = copyAsPicBuilder(builder, outputName, outputExtension); + cppConfiguration.getFdoSupport().configureCompilation(picBuilder, ruleContext, env, + ruleContext.getLabel(), ccRelativeName, nocopts, /*usePic=*/true, + lipoProvider); + + if (maySaveTemps) { + result.addTemps( + createTempsActions(sourceArtifact, outputName, picBuilder, /*usePic=*/true)); + } + + if (isCodeCoverageEnabled()) { + picBuilder.setGcnoFile(ruleContext.getRelatedArtifact(outputName, ".pic.gcno")); + } + + semantics.finalizeCompileActionBuilder(ruleContext, picBuilder); + CppCompileAction picAction = picBuilder.build(); + env.registerAction(picAction); + result.addPicObjectFile(picAction.getOutputFile()); + if (picAction.getDwoFile() != null) { + // Host targets don't produce .dwo files. + result.addPicDwoFile(picAction.getDwoFile()); + } + if (cppConfiguration.isLipoContextCollector() && !generateNoPicAction) { + result.addLipoScannable(picAction); + } + } + + if (generateNoPicAction) { + builder + .setOutputFile(ruleContext.getRelatedArtifact(outputName, outputExtension)) + .setDotdFile(outputName, ".d", ruleContext); + // Create non-PIC compile actions + cppConfiguration.getFdoSupport().configureCompilation(builder, ruleContext, env, + ruleContext.getLabel(), ccRelativeName, nocopts, /*usePic=*/false, + lipoProvider); + + if (maySaveTemps) { + result.addTemps( + createTempsActions(sourceArtifact, outputName, builder, /*usePic=*/false)); + } + + if (!cppConfiguration.isLipoOptimization() && isCodeCoverageEnabled()) { + builder.setGcnoFile(ruleContext.getRelatedArtifact(outputName, ".gcno")); + } + + semantics.finalizeCompileActionBuilder(ruleContext, builder); + CppCompileAction compileAction = builder.build(); + env.registerAction(compileAction); + Artifact objectFile = compileAction.getOutputFile(); + result.addObjectFile(objectFile); + if (compileAction.getDwoFile() != null) { + // Host targets don't produce .dwo files. + result.addDwoFile(compileAction.getDwoFile()); + } + if (cppConfiguration.isLipoContextCollector()) { + result.addLipoScannable(compileAction); + } + } + } + } + + /** + * Constructs the C++ linker actions. It generally generates two actions, one for a static library + * and one for a dynamic library. If PIC is required for shared libraries, but not for binaries, + * it additionally creates a third action to generate a PIC static library. + * + * <p>For dynamic libraries, this method can additionally create an interface shared library that + * can be used for linking, but doesn't contain any executable code. This increases the number of + * cache hits for link actions. Call {@link #setAllowInterfaceSharedObjects(boolean)} to enable + * this behavior. + */ + public CcLinkingOutputs createCcLinkActions(CcCompilationOutputs ccOutputs) { + // For now only handle static links. Note that the dynamic library link below ignores linkType. + // TODO(bazel-team): Either support non-static links or move this check to setLinkType(). + Preconditions.checkState(linkType.isStaticLibraryLink(), "can only handle static links"); + + CcLinkingOutputs.Builder result = new CcLinkingOutputs.Builder(); + if (cppConfiguration.isLipoContextCollector()) { + // Don't try to create LIPO link actions in collector mode, + // because it needs some data that's not available at this point. + return result.build(); + } + + AnalysisEnvironment env = ruleContext.getAnalysisEnvironment(); + boolean usePicForBinaries = CppHelper.usePic(ruleContext, true); + boolean usePicForSharedLibs = CppHelper.usePic(ruleContext, false); + + // Create static library (.a). The linkType only reflects whether the library is alwayslink or + // not. The PIC-ness is determined by whether we need to use PIC or not. There are three cases + // for (usePicForSharedLibs usePicForBinaries): + // + // (1) (false false) -> no pic code + // (2) (true false) -> shared libraries as pic, but not binaries + // (3) (true true) -> both shared libraries and binaries as pic + // + // In case (3), we always need PIC, so only create one static library containing the PIC object + // files. The name therefore does not match the content. + // + // Presumably, it is done this way because the .a file is an implicit output of every cc_library + // rule, so we can't use ".pic.a" that in the always-PIC case. + PathFragment linkedFileName = CppHelper.getLinkedFilename(ruleContext, linkType); + CppLinkAction maybePicAction = newLinkActionBuilder(linkedFileName) + .addNonLibraryInputs(ccOutputs.getObjectFiles(usePicForBinaries)) + .addNonLibraryInputs(ccOutputs.getHeaderTokenFiles()) + .setLinkType(linkType) + .setLinkStaticness(LinkStaticness.FULLY_STATIC) + .build(); + env.registerAction(maybePicAction); + result.addStaticLibrary(maybePicAction.getOutputLibrary()); + + // Create a second static library (.pic.a). Only in case (2) do we need both PIC and non-PIC + // static libraries. In that case, the first static library contains the non-PIC code, and this + // one contains the PIC code, so the names match the content. + if (!usePicForBinaries && usePicForSharedLibs) { + LinkTargetType picLinkType = (linkType == LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY) + ? LinkTargetType.ALWAYS_LINK_PIC_STATIC_LIBRARY + : LinkTargetType.PIC_STATIC_LIBRARY; + + PathFragment picFileName = CppHelper.getLinkedFilename(ruleContext, picLinkType); + CppLinkAction picAction = newLinkActionBuilder(picFileName) + .addNonLibraryInputs(ccOutputs.getObjectFiles(true)) + .addNonLibraryInputs(ccOutputs.getHeaderTokenFiles()) + .setLinkType(picLinkType) + .setLinkStaticness(LinkStaticness.FULLY_STATIC) + .build(); + env.registerAction(picAction); + result.addPicStaticLibrary(picAction.getOutputLibrary()); + } + + if (!createDynamicLibrary) { + return result.build(); + } + + // Create dynamic library. + if (soImplFilename == null) { + soImplFilename = CppHelper.getLinkedFilename(ruleContext, LinkTargetType.DYNAMIC_LIBRARY); + } + List<String> sonameLinkopts = ImmutableList.of(); + PathFragment soInterfaceFilename = null; + if (cppConfiguration.useInterfaceSharedObjects() && allowInterfaceSharedObjects) { + soInterfaceFilename = + CppHelper.getLinkedFilename(ruleContext, LinkTargetType.INTERFACE_DYNAMIC_LIBRARY); + Artifact dynamicLibrary = env.getDerivedArtifact( + soImplFilename, configuration.getBinDirectory()); + sonameLinkopts = ImmutableList.of("-Wl,-soname=" + + SolibSymlinkAction.getDynamicLibrarySoname(dynamicLibrary.getRootRelativePath(), false)); + } + + // Should we also link in any libraries that this library depends on? + // That is required on some systems... + CppLinkAction action = newLinkActionBuilder(soImplFilename) + .setInterfaceOutputPath(soInterfaceFilename) + .addNonLibraryInputs(ccOutputs.getObjectFiles(usePicForSharedLibs)) + .addNonLibraryInputs(ccOutputs.getHeaderTokenFiles()) + .setLinkType(LinkTargetType.DYNAMIC_LIBRARY) + .setLinkStaticness(LinkStaticness.DYNAMIC) + .addLinkopts(linkopts) + .addLinkopts(sonameLinkopts) + .setRuntimeInputs( + CppHelper.getToolchain(ruleContext).getDynamicRuntimeLinkMiddleman(), + CppHelper.getToolchain(ruleContext).getDynamicRuntimeLinkInputs()) + .build(); + env.registerAction(action); + + LibraryToLink dynamicLibrary = action.getOutputLibrary(); + LibraryToLink interfaceLibrary = action.getInterfaceOutputLibrary(); + if (interfaceLibrary == null) { + interfaceLibrary = dynamicLibrary; + } + + // If shared library has neverlink=1, then leave it untouched. Otherwise, + // create a mangled symlink for it and from now on reference it through + // mangled name only. + if (neverLink) { + result.addDynamicLibrary(interfaceLibrary); + result.addExecutionDynamicLibrary(dynamicLibrary); + } else { + LibraryToLink libraryLink = SolibSymlinkAction.getDynamicLibrarySymlink( + ruleContext, interfaceLibrary.getArtifact(), false, false, + ruleContext.getConfiguration()); + result.addDynamicLibrary(libraryLink); + LibraryToLink implLibraryLink = SolibSymlinkAction.getDynamicLibrarySymlink( + ruleContext, dynamicLibrary.getArtifact(), false, false, + ruleContext.getConfiguration()); + result.addExecutionDynamicLibrary(implLibraryLink); + } + return result.build(); + } + + private CppLinkAction.Builder newLinkActionBuilder(PathFragment outputPath) { + return new CppLinkAction.Builder(ruleContext, outputPath) + .setCrosstoolInputs(CppHelper.getToolchain(ruleContext).getLink()) + .addNonLibraryInputs(context.getCompilationPrerequisites()); + } + + /** + * Returns the output artifact path relative to the object directory. + */ + private PathFragment getObjectOutputPath(Artifact source, PathFragment objectDirectory) { + return objectDirectory.getRelative(semantics.getEffectiveSourcePath(source)); + } + + /** + * Creates a basic cpp compile action builder for source file. Configures options, + * crosstool inputs, output and dotd file names, compilation context and copts. + */ + private CppCompileActionBuilder createCompileActionBuilder( + Artifact source, Label label) { + CppCompileActionBuilder builder = new CppCompileActionBuilder( + ruleContext, source, label); + + builder + .setContext(context) + .addCopts(copts); + return builder; + } + + /** + * Creates cpp PIC compile action builder from the given builder by adding necessary copt and + * changing output and dotd file names. + */ + private CppCompileActionBuilder copyAsPicBuilder(CppCompileActionBuilder builder, + PathFragment outputName, String outputExtension) { + CppCompileActionBuilder picBuilder = new CppCompileActionBuilder(builder); + picBuilder.addCopt("-fPIC") + .setOutputFile(ruleContext.getRelatedArtifact(outputName, ".pic" + outputExtension)) + .setDotdFile(outputName, ".pic.d", ruleContext); + return picBuilder; + } + + /** + * Create the actions for "--save_temps". + */ + private ImmutableList<Artifact> createTempsActions(Artifact source, PathFragment outputName, + CppCompileActionBuilder builder, boolean usePic) { + if (!cppConfiguration.getSaveTemps()) { + return ImmutableList.of(); + } + + String path = source.getFilename(); + boolean isCFile = CppFileTypes.C_SOURCE.matches(path); + boolean isCppFile = CppFileTypes.CPP_SOURCE.matches(path); + + if (!isCFile && !isCppFile) { + return ImmutableList.of(); + } + + String iExt = isCFile ? ".i" : ".ii"; + String picExt = usePic ? ".pic" : ""; + CppCompileActionBuilder dBuilder = new CppCompileActionBuilder(builder); + CppCompileActionBuilder sdBuilder = new CppCompileActionBuilder(builder); + + dBuilder + .setOutputFile(ruleContext.getRelatedArtifact(outputName, picExt + iExt)) + .setDotdFile(outputName, picExt + iExt + ".d", ruleContext); + semantics.finalizeCompileActionBuilder(ruleContext, dBuilder); + CppCompileAction dAction = dBuilder.build(); + ruleContext.registerAction(dAction); + + sdBuilder + .setOutputFile(ruleContext.getRelatedArtifact(outputName, picExt + ".s")) + .setDotdFile(outputName, picExt + ".s.d", ruleContext); + semantics.finalizeCompileActionBuilder(ruleContext, sdBuilder); + CppCompileAction sdAction = sdBuilder.build(); + ruleContext.registerAction(sdAction); + return ImmutableList.of( + dAction.getOutputFile(), + sdAction.getOutputFile()); + } + + /** + * Returns true iff code coverage is enabled for the given target. + */ + private boolean isCodeCoverageEnabled() { + if (configuration.isCodeCoverageEnabled()) { + final RegexFilter filter = configuration.getInstrumentationFilter(); + // If rule is matched by the instrumentation filter, enable instrumentation + if (filter.isIncluded(ruleContext.getLabel().toString())) { + return true; + } + // At this point the rule itself is not matched by the instrumentation filter. However, we + // might still want to instrument C++ rules if one of the targets listed in "deps" is + // instrumented and, therefore, can supply header files that we would want to collect code + // coverage for. For example, think about cc_test rule that tests functionality defined in a + // header file that is supplied by the cc_library. + // + // Note that we only check direct prerequisites and not the transitive closure. This is done + // for two reasons: + // a) It is a good practice to declare libraries which you directly rely on. Including headers + // from a library hidden deep inside the transitive closure makes build dependencies less + // readable and can lead to unexpected breakage. + // b) Traversing the transitive closure for each C++ compile action would require more complex + // implementation (with caching results of this method) to avoid O(N^2) slowdown. + if (ruleContext.getRule().isAttrDefined("deps", Type.LABEL_LIST)) { + for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps", Mode.TARGET)) { + if (dep.getProvider(CppCompilationContext.class) != null + && filter.isIncluded(dep.getLabel().toString())) { + return true; + } + } + } + } + return false; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMap.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMap.java new file mode 100644 index 0000000000..bb272098a4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMap.java @@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Structure for C++ module maps. Stores the name of the module and a .cppmap artifact. + */ +@Immutable +public class CppModuleMap { + private final Artifact artifact; + private final String name; + + public CppModuleMap(Artifact artifact, String name) { + this.artifact = artifact; + this.name = name; + } + + public Artifact getArtifact() { + return artifact; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return name + "@" + artifact; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMapAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMapAction.java new file mode 100644 index 0000000000..a350fc4618 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMapAction.java @@ -0,0 +1,185 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Creates C++ module map artifact genfiles. These are then passed to Clang to + * do dependency checking. + */ +public class CppModuleMapAction extends AbstractFileWriteAction { + + private static final String GUID = "4f407081-1951-40c1-befc-d6b4daff5de3"; + + // C++ module map of the current target + private final CppModuleMap cppModuleMap; + + /** + * If set, the paths in the module map are relative to the current working directory instead + * of relative to the module map file's location. + */ + private final boolean moduleMapHomeIsCwd; + + // Headers and dependencies list + private final ImmutableList<Artifact> privateHeaders; + private final ImmutableList<Artifact> publicHeaders; + private final ImmutableList<CppModuleMap> dependencies; + private final ImmutableList<PathFragment> additionalExportedHeaders; + private final boolean compiledModule; + + public CppModuleMapAction(ActionOwner owner, CppModuleMap cppModuleMap, + Iterable<Artifact> privateHeaders, Iterable<Artifact> publicHeaders, + Iterable<CppModuleMap> dependencies, Iterable<PathFragment> additionalExportedHeaders, + boolean compiledModule, boolean moduleMapHomeIsCwd) { + super(owner, ImmutableList.<Artifact>of(), cppModuleMap.getArtifact(), + /*makeExecutable=*/false); + this.cppModuleMap = cppModuleMap; + this.moduleMapHomeIsCwd = moduleMapHomeIsCwd; + this.privateHeaders = ImmutableList.copyOf(privateHeaders); + this.publicHeaders = ImmutableList.copyOf(publicHeaders); + this.dependencies = ImmutableList.copyOf(dependencies); + this.additionalExportedHeaders = ImmutableList.copyOf(additionalExportedHeaders); + this.compiledModule = compiledModule; + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) { + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + StringBuilder content = new StringBuilder(); + PathFragment fragment = cppModuleMap.getArtifact().getExecPath(); + int segmentsToExecPath = fragment.segmentCount() - 1; + + // For details about the different header types, see: + // http://clang.llvm.org/docs/Modules.html#header-declaration + String leadingPeriods = moduleMapHomeIsCwd ? "" : Strings.repeat("../", segmentsToExecPath); + content.append("module \"").append(cppModuleMap.getName()).append("\" {\n"); + content.append(" export *\n"); + for (Artifact artifact : privateHeaders) { + appendHeader(content, "private", artifact.getExecPath(), leadingPeriods, + /*canCompile=*/true); + } + for (Artifact artifact : publicHeaders) { + appendHeader(content, "", artifact.getExecPath(), leadingPeriods, /*canCompile=*/true); + } + for (PathFragment additionalExportedHeader : additionalExportedHeaders) { + appendHeader(content, "", additionalExportedHeader, leadingPeriods, /*canCompile*/false); + } + for (CppModuleMap dep : dependencies) { + content.append(" use \"").append(dep.getName()).append("\"\n"); + } + content.append("}"); + for (CppModuleMap dep : dependencies) { + content.append("\nextern module \"") + .append(dep.getName()) + .append("\" \"") + .append(leadingPeriods) + .append(dep.getArtifact().getExecPath()) + .append("\""); + } + out.write(content.toString().getBytes(StandardCharsets.ISO_8859_1)); + } + }; + } + + private void appendHeader(StringBuilder content, String visibilitySpecifier, PathFragment path, + String leadingPeriods, boolean canCompile) { + content.append(" "); + if (!visibilitySpecifier.isEmpty()) { + content.append(visibilitySpecifier).append(" "); + } + if (!canCompile || !shouldCompileHeader(path)) { + content.append("textual "); + } + content.append("header \"").append(leadingPeriods).append(path).append("\"\n"); + } + + private boolean shouldCompileHeader(PathFragment path) { + return compiledModule && !CppFileTypes.CPP_TEXTUAL_INCLUDE.matches(path); + } + + @Override + public String getMnemonic() { + return "CppModuleMap"; + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addInt(privateHeaders.size()); + for (Artifact artifact : privateHeaders) { + f.addPath(artifact.getRootRelativePath()); + } + f.addInt(publicHeaders.size()); + for (Artifact artifact : publicHeaders) { + f.addPath(artifact.getRootRelativePath()); + } + f.addInt(dependencies.size()); + for (CppModuleMap dep : dependencies) { + f.addPath(dep.getArtifact().getExecPath()); + } + f.addPath(cppModuleMap.getArtifact().getExecPath()); + f.addString(cppModuleMap.getName()); + return f.hexDigestAndReset(); + } + + @Override + public ResourceSet estimateResourceConsumptionLocal() { + return new ResourceSet(/*memoryMb=*/0, /*cpuUsage=*/0, /*ioUsage=*/0.02); + } + + @VisibleForTesting + public Collection<Artifact> getPublicHeaders() { + return publicHeaders; + } + + @VisibleForTesting + public Collection<Artifact> getPrivateHeaders() { + return privateHeaders; + } + + @VisibleForTesting + public ImmutableList<PathFragment> getAdditionalExportedHeaders() { + return additionalExportedHeaders; + } + + @VisibleForTesting + public Collection<Artifact> getDependencyArtifacts() { + List<Artifact> artifacts = new ArrayList<>(); + for (CppModuleMap map : dependencies) { + artifacts.add(map.getArtifact()); + } + return artifacts; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java new file mode 100644 index 0000000000..b0f2e820a5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java @@ -0,0 +1,646 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.LabelConverter; +import com.google.devtools.build.lib.analysis.config.CompilationMode; +import com.google.devtools.build.lib.analysis.config.FragmentOptions; +import com.google.devtools.build.lib.analysis.config.PerLabelOptions; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.HeadersCheckingMode; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.LibcTop; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.StripMode; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.EnumConverter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsParsingException; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * Command-line options for C++. + */ +public class CppOptions extends FragmentOptions { + /** + * Label of a filegroup that contains all crosstool files for all configurations. + */ + @VisibleForTesting + public static final String DEFAULT_CROSSTOOL_TARGET = "//tools/cpp:toolchain"; + + + /** + * Converter for --cwarn flag + */ + public static class GccWarnConverter implements Converter<String> { + @Override + public String convert(String input) throws OptionsParsingException { + if (input.startsWith("no-") || input.startsWith("-W")) { + throw new OptionsParsingException("Not a valid gcc warning to enable"); + } + return input; + } + + @Override + public String getTypeDescription() { + return "A gcc warning to enable"; + } + } + + /** + * Converts a comma-separated list of compilation mode settings to a properly typed List. + */ + public static class FissionOptionConverter implements Converter<List<CompilationMode>> { + @Override + public List<CompilationMode> convert(String input) throws OptionsParsingException { + ImmutableSet.Builder<CompilationMode> modes = ImmutableSet.builder(); + if (input.equals("yes")) { // Special case: enable all modes. + modes.add(CompilationMode.values()); + } else if (!input.equals("no")) { // "no" is another special case that disables all modes. + CompilationMode.Converter modeConverter = new CompilationMode.Converter(); + for (String mode : Splitter.on(',').split(input)) { + modes.add(modeConverter.convert(mode)); + } + } + return modes.build().asList(); + } + + @Override + public String getTypeDescription() { + return "a set of compilation modes"; + } + } + + /** + * The same as DynamicMode, but on command-line we also allow AUTO. + */ + public enum DynamicModeFlag { OFF, DEFAULT, FULLY, AUTO } + + /** + * Converter for DynamicModeFlag + */ + public static class DynamicModeConverter extends EnumConverter<DynamicModeFlag> { + public DynamicModeConverter() { + super(DynamicModeFlag.class, "dynamic mode"); + } + } + + /** + * Converter for the --strip option. + */ + public static class StripModeConverter extends EnumConverter<StripMode> { + public StripModeConverter() { + super(StripMode.class, "strip mode"); + } + } + + private static final String LIBC_RELATIVE_LABEL = ":everything"; + + /** + * Converts a String, which is an absolute path or label into a LibcTop + * object. + */ + public static class LibcTopConverter implements Converter<LibcTop> { + @Override + public LibcTop convert(String input) throws OptionsParsingException { + if (!input.startsWith("//")) { + throw new OptionsParsingException("Not a label"); + } + try { + Label label = Label.parseAbsolute(input).getRelative(LIBC_RELATIVE_LABEL); + return new LibcTop(label); + } catch (SyntaxException e) { + throw new OptionsParsingException(e.getMessage()); + } + } + + @Override + public String getTypeDescription() { + return "a label"; + } + } + + /** + * Converter for the --hdrs_check option. + */ + public static class HdrsCheckConverter extends EnumConverter<HeadersCheckingMode> { + public HdrsCheckConverter() { + super(HeadersCheckingMode.class, "Headers check mode"); + } + } + + /** + * Checks whether a string is a valid regex pattern and compiles it. + */ + public static class NullableRegexPatternConverter implements Converter<Pattern> { + + @Override + public Pattern convert(String input) throws OptionsParsingException { + if (input.isEmpty()) { + return null; + } + try { + return Pattern.compile(input); + } catch (PatternSyntaxException e) { + throw new OptionsParsingException("Not a valid regular expression: " + e.getMessage()); + } + } + + @Override + public String getTypeDescription() { + return "a valid Java regular expression"; + } + } + + /** + * Converter for the --lipo option. + */ + public static class LipoModeConverter extends EnumConverter<LipoMode> { + public LipoModeConverter() { + super(LipoMode.class, "LIPO mode"); + } + } + + @Option(name = "lipo input collector", + defaultValue = "false", + category = "undocumented", + help = "Internal flag, only used to create configurations with the LIPO-collector flag set.") + public boolean lipoCollector; + + @Option(name = "crosstool_top", + defaultValue = CppOptions.DEFAULT_CROSSTOOL_TARGET, + category = "version", + converter = LabelConverter.class, + help = "The label of the crosstool package to be used for compiling C++ code.") + public Label crosstoolTop; + + @Option(name = "compiler", + defaultValue = "null", + category = "version", + help = "The C++ compiler to use for compiling the target.") + public String cppCompiler; + + @Option(name = "glibc", + defaultValue = "null", + category = "version", + help = "The version of glibc the target should be linked against. " + + "By default, a suitable version is chosen based on --cpu.") + public String glibc; + + @Option(name = "thin_archives", + defaultValue = "false", + category = "strategy", // but also adds edges to the action graph + help = "Pass the 'T' flag to ar if supported by the toolchain. " + + "All supported toolchains support this setting.") + public boolean useThinArchives; + + // O intrepid reaper of unused options: Be warned that the [no]start_end_lib + // option, however tempting to remove, has a use case. Look in our telemetry data. + @Option(name = "start_end_lib", + defaultValue = "true", + category = "strategy", // but also adds edges to the action graph + help = "Use the --start-lib/--end-lib ld options if supported by the toolchain.") + public boolean useStartEndLib; + + @Option(name = "interface_shared_objects", + defaultValue = "true", + category = "strategy", // but also adds edges to the action graph + help = "Use interface shared objects if supported by the toolchain. " + + "All ELF toolchains currently support this setting.") + public boolean useInterfaceSharedObjects; + + @Option(name = "cc_include_scanning", + defaultValue = "true", + category = "strategy", + help = "Whether to perform include scanning. Without it, your build will most likely " + + "fail.") + public boolean scanIncludes; + + @Option(name = "extract_generated_inclusions", + defaultValue = "true", + category = "undocumented", + help = "Run grep-includes actions (used for include scanning) over " + + "generated headers and sources.") + public boolean extractInclusions; + + @Option(name = "fission", + defaultValue = "no", + converter = FissionOptionConverter.class, + category = "semantics", + help = "Specifies which compilation modes use fission for C++ compilations and links. " + + " May be any combination of {'fastbuild', 'dbg', 'opt'} or the special values 'yes' " + + " to enable all modes and 'no' to disable all modes.") + public List<CompilationMode> fissionModes; + + @Option(name = "dynamic_mode", + defaultValue = "default", + converter = DynamicModeConverter.class, + category = "semantics", + help = "Determines whether C++ binaries will be linked dynamically. 'default' means " + + "blaze will choose whether to link dynamically. 'fully' means all libraries " + + "will be linked dynamically. 'off' means that all libraries will be linked " + + "in mostly static mode.") + public DynamicModeFlag dynamicMode; + + @Option(name = "force_pic", + defaultValue = "false", + category = "semantics", + help = "If enabled, all C++ compilations produce position-independent code (\"-fPIC\")," + + " links prefer PIC pre-built libraries over non-PIC libraries, and links produce" + + " position-independent executables (\"-pie\").") + public boolean forcePic; + + @Option(name = "force_ignore_dash_static", + defaultValue = "false", + category = "semantics", + help = "If set, '-static' options in the linkopts of cc_* rules will be ignored.") + public boolean forceIgnoreDashStatic; + + @Option(name = "experimental_skip_static_outputs", + defaultValue = "false", + category = "semantics", + help = "This flag is experimental and may go away at any time. " + + "If true, linker output for mostly-static C++ executables is a tiny amount of " + + "dummy dependency information, and NOT a usable binary. Kludge, but can reduce " + + "network and disk I/O load (and thus, continuous build cycle times) by a lot. " + + "NOTE: use of this flag REQUIRES --distinct_host_configuration.") + public boolean skipStaticOutputs; + + @Option(name = "hdrs_check", + allowMultiple = false, + defaultValue = "loose", + converter = HdrsCheckConverter.class, + category = "semantics", + help = "Headers check mode for rules that don't specify it explicitly using a " + + "hdrs_check attribute. Allowed values: 'loose' allows undeclared headers, 'warn' " + + "warns about undeclared headers, and 'strict' disallows them.") + public HeadersCheckingMode headersCheckingMode; + + @Option(name = "copt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to gcc.") + public List<String> coptList; + + @Option(name = "cwarn", + converter = GccWarnConverter.class, + defaultValue = "", + category = "flags", + allowMultiple = true, + help = "Additional warnings to enable when compiling C or C++ source files.") + public List<String> cWarns; + + @Option(name = "cxxopt", + defaultValue = "", + category = "flags", + allowMultiple = true, + help = "Additional option to pass to gcc when compiling C++ source files.") + public List<String> cxxoptList; + + @Option(name = "conlyopt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional option to pass to gcc when compiling C source files.") + public List<String> conlyoptList; + + @Option(name = "linkopt", + defaultValue = "", + category = "flags", + allowMultiple = true, + help = "Additional option to pass to gcc when linking.") + public List<String> linkoptList; + + @Option(name = "stripopt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to strip when generating a '<name>.stripped' binary.") + public List<String> stripoptList; + + @Option(name = "custom_malloc", + defaultValue = "null", + category = "semantics", + help = "Specifies a custom malloc implementation. This setting overrides malloc " + + "attributes in build rules.", + converter = LabelConverter.class) + public Label customMalloc; + + @Option(name = "cpp_module_maps", + defaultValue = "true", + category = "flags", + help = "If true then C++ targets create a module map based on BUILD files, and " + + "pass them to the compiler.") + public boolean cppModuleMaps; + + @Option(name = "legacy_whole_archive", + defaultValue = "true", + category = "semantics", + help = "When on, use --whole-archive for cc_binary rules that have " + + "linkshared=1 and either linkstatic=1 or '-static' in linkopts. " + + "This is for backwards compatibility only. " + + "A better alternative is to use alwayslink=1 where required.") + public boolean legacyWholeArchive; + + @Option(name = "strip", + defaultValue = "sometimes", + category = "flags", + help = "Specifies whether to strip binaries and shared libraries " + + " (using \"-Wl,--strip-debug\"). The default value of 'sometimes'" + + " means strip iff --compilation_mode=fastbuild.", + converter = StripModeConverter.class) + public StripMode stripBinaries; + + @Option(name = "fdo_instrument", + defaultValue = "null", + converter = OptionsUtils.PathFragmentConverter.class, + category = "flags", + implicitRequirements = {"--copt=-Wno-error"}, + help = "Generate binaries with FDO instrumentation. Specify the relative " + + "directory name for the .gcda files at runtime.") + public PathFragment fdoInstrument; + + @Option(name = "fdo_optimize", + defaultValue = "null", + category = "flags", + help = "Use FDO profile information to optimize compilation. Specify the name " + + "of the zip file containing the .gcda file tree or an afdo file containing " + + "an auto profile. This flag also accepts files specified as labels, for " + + "example //foo/bar:file.afdo. Such labels must refer to input files; you may " + + "need to add an exports_files directive to the corresponding package to make " + + "the file visible to Blaze.") + public String fdoOptimize; + + @Option(name = "autofdo_lipo_data", + defaultValue = "false", + category = "flags", + help = "If true then the directory name for non-LIPO targets will have a " + + "'-lipodata' suffix in AutoFDO mode.") + public boolean autoFdoLipoData; + + @Option(name = "lipo", + defaultValue = "off", + converter = LipoModeConverter.class, + category = "flags", + help = "Enable LIPO optimization (lightweight inter-procedural optimization, The allowed " + + "values for this option are 'off' and 'binary', which enables LIPO. This option only " + + "has an effect when FDO is also enabled. Currently LIPO is only supported when " + + "building a single cc_binary rule.") + public LipoMode lipoMode; + + @Option(name = "lipo_context", + defaultValue = "null", + category = "flags", + converter = LabelConverter.class, + implicitRequirements = {"--linkopt=-Wl,--warn-unresolved-symbols"}, + help = "Specifies the binary from which the LIPO profile information comes.") + public Label lipoContext; + + @Option(name = "experimental_stl", + converter = LabelConverter.class, + defaultValue = "null", + category = "version", + help = "If set, use this label instead of the default STL implementation. " + + "This option is EXPERIMENTAL and may go away in a future release.") + public Label stl; + + @Option(name = "save_temps", + defaultValue = "false", + category = "what", + help = "If set, temporary outputs from gcc will be saved. " + + "These include .s files (assembler code), .i files (preprocessed C) and " + + ".ii files (preprocessed C++).") + public boolean saveTemps; + + @Option(name = "per_file_copt", + allowMultiple = true, + converter = PerLabelOptions.PerLabelOptionsConverter.class, + defaultValue = "", + category = "semantics", + help = "Additional options to selectively pass to gcc when compiling certain files. " + + "This option can be passed multiple times. " + + "Syntax: regex_filter@option_1,option_2,...,option_n. Where regex_filter stands " + + "for a list of include and exclude regular expression patterns (Also see " + + "--instrumentation_filter). option_1 to option_n stand for " + + "arbitrary command line options. If an option contains a comma it has to be " + + "quoted with a backslash. Options can contain @. Only the first @ is used to " + + "split the string. Example: " + + "--per_file_copt=//foo/.*\\.cc,-//foo/bar\\.cc@-O0 adds the -O0 " + + "command line option to the gcc command line of all cc files in //foo/ " + + "except bar.cc.") + public List<PerLabelOptions> perFileCopts; + + @Option(name = "host_crosstool_top", + defaultValue = "null", + converter = LabelConverter.class, + category = "semantics", + help = "By default, the --crosstool_top, --glibc, and --compiler options are also used " + + "for the host configuration. If this flag is provided, Blaze uses the default glibc " + + "and compiler for the given crosstool_top.") + public Label hostCrosstoolTop; + + @Option(name = "host_copt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to gcc for host tools.") + public List<String> hostCoptList; + + @Option(name = "define", + converter = Converters.AssignmentConverter.class, + defaultValue = "", + category = "semantics", + allowMultiple = true, + help = "Each --define option specifies an assignment for a build variable.") + public List<Map.Entry<String, String>> commandLineDefinedVariables; + + @Option(name = "grte_top", + defaultValue = "null", // The default value is chosen by the toolchain. + category = "version", + converter = LibcTopConverter.class, + help = "A label to a checked-in libc library. The default value is selected by the crosstool " + + "toolchain, and you almost never need to override it.") + public LibcTop libcTop; + + @Option(name = "host_grte_top", + defaultValue = "null", // The default value is chosen by the toolchain. + category = "version", + converter = LibcTopConverter.class, + help = "If specified, this setting overrides the libc top-level directory (--grte_top) " + + "for the host configuration.") + public LibcTop hostLibcTop; + + @Option(name = "output_symbol_counts", + defaultValue = "false", + category = "flags", + help = "If enabled, every C++ binary linked with gold will store the number of used " + + "symbols per object file in a .sc file.") + public boolean symbolCounts; + + @Option(name = "experimental_inmemory_dotd_files", + defaultValue = "false", + category = "experimental", + help = "If enabled, C++ .d files will be passed through in memory directly from the remote " + + "build nodes instead of being written to disk.") + public boolean inmemoryDotdFiles; + + @Option(name = "use_isystem_for_includes", + defaultValue = "true", + category = "undocumented", + help = "Instruct C and C++ compilations to treat 'includes' paths as system header " + + "paths, by translating it into -isystem instead of -I.") + public boolean useIsystemForIncludes; + + @Option(name = "experimental_omitfp", + defaultValue = "false", + category = "semantics", + help = "If true, use libunwind for stack unwinding, and compile with " + + "-fomit-frame-pointer and -fasynchronous-unwind-tables.") + public boolean experimentalOmitfp; + + @Option(name = "share_native_deps", + defaultValue = "true", + category = "strategy", + help = "If true, native libraries that contain identical functionality " + + "will be shared among different targets") + public boolean shareNativeDeps; + + @Override + public FragmentOptions getHost(boolean fallback) { + CppOptions host = (CppOptions) getDefault(); + + host.commandLineDefinedVariables = commandLineDefinedVariables; + + // The crosstool options are partially copied from the target configuration. + if (!fallback) { + if (hostCrosstoolTop == null) { + host.cppCompiler = cppCompiler; + host.crosstoolTop = crosstoolTop; + host.glibc = glibc; + } else { + host.crosstoolTop = hostCrosstoolTop; + } + } + + if (hostLibcTop != null) { + host.libcTop = hostLibcTop; + } else if (hostCrosstoolTop == null) { + // Track libc in the host configuration if no host crosstool is set. + host.libcTop = libcTop; + } + + // -g0 is the default, but allowMultiple options cannot have default values so we just pass + // -g0 first and let the user options override it. + host.coptList = ImmutableList.<String>builder().add("-g0").addAll(hostCoptList).build(); + + host.useThinArchives = useThinArchives; + host.useStartEndLib = useStartEndLib; + host.extractInclusions = extractInclusions; + host.stripBinaries = StripMode.ALWAYS; + host.fdoOptimize = null; + host.lipoMode = LipoMode.OFF; + host.scanIncludes = scanIncludes; + host.inmemoryDotdFiles = inmemoryDotdFiles; + host.cppModuleMaps = cppModuleMaps; + + return host; + } + + @Override + public void addAllLabels(Multimap<String, Label> labelMap) { + labelMap.put("crosstool", crosstoolTop); + if (hostCrosstoolTop != null) { + labelMap.put("crosstool", hostCrosstoolTop); + } + + if (libcTop != null) { + Label libcLabel = libcTop.getLabel(); + if (libcLabel != null) { + labelMap.put("crosstool", libcLabel); + } + } + addOptionalLabel(labelMap, "fdo", fdoOptimize); + + if (stl != null) { + labelMap.put("STL", stl); + } + + if (customMalloc != null) { + labelMap.put("custom_malloc", customMalloc); + } + + if (getLipoContextLabel() != null) { + labelMap.put("lipo", getLipoContextLabel()); + } + } + + @Override + public Map<String, Set<Label>> getDefaultsLabels(BuildConfiguration.Options commonOptions) { + Set<Label> crosstoolLabels = new LinkedHashSet<>(); + crosstoolLabels.add(crosstoolTop); + if (hostCrosstoolTop != null) { + crosstoolLabels.add(hostCrosstoolTop); + } + + if (libcTop != null) { + Label libcLabel = libcTop.getLabel(); + if (libcLabel != null) { + crosstoolLabels.add(libcLabel); + } + } + + return ImmutableMap.of( + "CROSSTOOL", crosstoolLabels, + "COVERAGE", ImmutableSet.<Label>of()); + } + + public boolean isFdo() { + return fdoOptimize != null || fdoInstrument != null; + } + + public boolean isLipoOptimization() { + return lipoMode == LipoMode.BINARY && fdoOptimize != null && lipoContext != null; + } + + public boolean isLipoOptimizationOrInstrumentation() { + return lipoMode == LipoMode.BINARY && + ((fdoOptimize != null && lipoContext != null) || fdoInstrument != null); + } + + public Label getLipoContextLabel() { + return (lipoMode == LipoMode.BINARY && fdoOptimize != null) + ? lipoContext : null; + } + + public LipoMode getLipoMode() { + return lipoMode; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java new file mode 100644 index 0000000000..de7c95d4a8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java @@ -0,0 +1,104 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ALWAYS_LINK_LIBRARY; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ALWAYS_LINK_PIC_LIBRARY; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ARCHIVE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.CPP_HEADER; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.CPP_SOURCE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.C_SOURCE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.OBJECT_FILE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.PIC_ARCHIVE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.PIC_OBJECT_FILE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.SHARED_LIBRARY; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.VERSIONED_SHARED_LIBRARY; + +import com.google.devtools.build.lib.analysis.LanguageDependentFragment.LibraryLanguage; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec; +import com.google.devtools.build.lib.util.FileTypeSet; + +/** + * Rule class definitions for C++ rules. + */ +public class CppRuleClasses { + // Artifacts of these types are discarded from the 'hdrs' attribute in cc rules + static final FileTypeSet DISALLOWED_HDRS_FILES = FileTypeSet.of( + ARCHIVE, + PIC_ARCHIVE, + ALWAYS_LINK_LIBRARY, + ALWAYS_LINK_PIC_LIBRARY, + SHARED_LIBRARY, + VERSIONED_SHARED_LIBRARY, + OBJECT_FILE, + PIC_OBJECT_FILE); + + /** + * The set of instrumented source file types; keep this in sync with the list above. Note that + * extension-less header files cannot currently be declared, so we cannot collect coverage for + * those. + */ + static final InstrumentationSpec INSTRUMENTATION_SPEC = new InstrumentationSpec( + FileTypeSet.of(CPP_SOURCE, C_SOURCE, CPP_HEADER, ASSEMBLER_WITH_C_PREPROCESSOR), + "srcs", "deps", "data", "hdrs", "implements", "implementation"); + + public static final LibraryLanguage LANGUAGE = new LibraryLanguage("C++"); + + /** + * Implicit outputs for cc_binary rules. + */ + public static final SafeImplicitOutputsFunction CC_BINARY_STRIPPED = + fromTemplates("%{name}.stripped"); + + + // Used for requesting dwp "debug packages". + public static final SafeImplicitOutputsFunction CC_BINARY_DEBUG_PACKAGE = + fromTemplates("%{name}.dwp"); + + + /** + * Path of the build_interface_so script in the Blaze binary. + */ + public static final String BUILD_INTERFACE_SO = "build_interface_so"; + + /** + * A string constant for the layering_check feature. + */ + public static final String LAYERING_CHECK = "layering_check"; + + /** + * A string constant for the parse_headers feature. + */ + public static final String PARSE_HEADERS = "parse_headers"; + + /** + * A string constant for the preprocess_headers feature. + */ + public static final String PREPROCESS_HEADERS = "preprocess_headers"; + + /** + * A string constant for the header_modules feature. + */ + public static final String HEADER_MODULES = "header_modules"; + + /** + * A string constant for the module_map_home_cwd feature. + */ + public static final String MODULE_MAP_HOME_CWD = "module_map_home_cwd"; + +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRunfilesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRunfilesProvider.java new file mode 100644 index 0000000000..f4aa38cb1f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRunfilesProvider.java @@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Runfiles provider for C++ targets. + * + * <p>Contains two {@link Runfiles} objects: one for the eventual statically linked binary and + * one for the one that uses shared libraries. Data dependencies are present in both. + */ +@Immutable +public final class CppRunfilesProvider implements TransitiveInfoProvider { + private final Runfiles staticRunfiles; + private final Runfiles sharedRunfiles; + + public CppRunfilesProvider(Runfiles staticRunfiles, Runfiles sharedRunfiles) { + this.staticRunfiles = staticRunfiles; + this.sharedRunfiles = sharedRunfiles; + } + + public Runfiles getStaticRunfiles() { + return staticRunfiles; + } + + public Runfiles getSharedRunfiles() { + return sharedRunfiles; + } + + /** + * Returns a function that gets the static C++ runfiles from a {@link TransitiveInfoCollection} + * or the empty runfiles instance if it does not contain that provider. + */ + public static final Function<TransitiveInfoCollection, Runfiles> STATIC_RUNFILES = + new Function<TransitiveInfoCollection, Runfiles>() { + @Override + public Runfiles apply(TransitiveInfoCollection input) { + CppRunfilesProvider provider = input.getProvider(CppRunfilesProvider.class); + return provider == null + ? Runfiles.EMPTY + : provider.getStaticRunfiles(); + } + }; + + /** + * Returns a function that gets the shared C++ runfiles from a {@link TransitiveInfoCollection} + * or the empty runfiles instance if it does not contain that provider. + */ + public static final Function<TransitiveInfoCollection, Runfiles> SHARED_RUNFILES = + new Function<TransitiveInfoCollection, Runfiles>() { + @Override + public Runfiles apply(TransitiveInfoCollection input) { + CppRunfilesProvider provider = input.getProvider(CppRunfilesProvider.class); + return provider == null + ? Runfiles.EMPTY + : provider.getSharedRunfiles(); + } + }; + + /** + * Returns a function that gets the C++ runfiles from a {@link TransitiveInfoCollection} or + * the empty runfiles instance if it does not contain that provider. + */ + public static final Function<TransitiveInfoCollection, Runfiles> runfilesFunction( + boolean linkingStatically) { + return linkingStatically ? STATIC_RUNFILES : SHARED_RUNFILES; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppSemantics.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppSemantics.java new file mode 100644 index 0000000000..600b2fa342 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppSemantics.java @@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Pluggable C++ compilation semantics. + */ +public interface CppSemantics { + /** + * Returns the "effective source path" of a source file. + * + * <p>It is used, among other things, for computing the output path. + */ + PathFragment getEffectiveSourcePath(Artifact source); + + /** + * Called before a C++ compile action is built. + * + * <p>Gives the semantics implementation the opportunity to change compile actions at the last + * minute. + */ + void finalizeCompileActionBuilder( + RuleContext ruleContext, CppCompileActionBuilder actionBuilder); + + /** + * Called before {@link CppCompilationContext}s are finalized. + * + * <p>Gives the semantics implementation the opportunity to change what the C++ rule propagates + * to dependent rules. + */ + void setupCompilationContext( + RuleContext ruleContext, CppCompilationContext.Builder contextBuilder); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationIdentifier.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationIdentifier.java new file mode 100644 index 0000000000..1111189101 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationIdentifier.java @@ -0,0 +1,132 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.CToolchain; + +import java.util.Objects; + +/** + * Contains parameters which uniquely describe a crosstool configuration + * and methods for comparing two crosstools against each other. + * + * <p>Two crosstools which contain equivalent values of these parameters are + * considered equal. + */ +public final class CrosstoolConfigurationIdentifier implements CrosstoolConfigurationOptions { + + /** The CPU associated with this crosstool configuration. */ + private final String cpu; + + /** The compiler (e.g. gcc) associated with this crosstool configuration. */ + private final String compiler; + + /** The version of libc (e.g. glibc-2.11) associated with this crosstool configuration. */ + private final String libc; + + private CrosstoolConfigurationIdentifier(String cpu, String compiler, String libc) { + this.cpu = cpu; + this.compiler = compiler; + this.libc = libc; + } + + /** + * Creates a new crosstool configuration from the given crosstool release and + * configuration options. + */ + public static CrosstoolConfigurationIdentifier fromReleaseAndCrosstoolConfiguration( + CrosstoolConfig.CrosstoolRelease release, BuildOptions buildOptions) { + String cpu = buildOptions.get(BuildConfiguration.Options.class).getCpu(); + if (cpu == null) { + cpu = release.getDefaultTargetCpu(); + } + CppOptions cppOptions = buildOptions.get(CppOptions.class); + return new CrosstoolConfigurationIdentifier(cpu, cppOptions.cppCompiler, cppOptions.glibc); + } + + public static CrosstoolConfigurationIdentifier fromToolchain(CToolchain toolchain) { + return new CrosstoolConfigurationIdentifier( + toolchain.getTargetCpu(), toolchain.getCompiler(), toolchain.getTargetLibc()); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof CrosstoolConfigurationIdentifier)) { + return false; + } + CrosstoolConfigurationIdentifier otherCrosstool = (CrosstoolConfigurationIdentifier) other; + return Objects.equals(cpu, otherCrosstool.cpu) + && Objects.equals(compiler, otherCrosstool.compiler) + && Objects.equals(libc, otherCrosstool.libc); + } + + @Override + public int hashCode() { + return Objects.hash(cpu, compiler, libc); + } + + + /** + * Returns a series of command line flags which specify the configuration options. + * Any of these options may be null, in which case its flag is omitted. + * + * <p>The appended string will be along the lines of + * " --cpu='cpu' --compiler='compiler' --glibc='libc'". + */ + public String describeFlags() { + StringBuilder message = new StringBuilder(); + if (getCpu() != null) { + message.append(" --cpu='").append(getCpu()).append("'"); + } + if (getCompiler() != null) { + message.append(" --compiler='").append(getCompiler()).append("'"); + } + if (getLibc() != null) { + message.append(" --glibc='").append(getLibc()).append("'"); + } + return message.toString(); + } + + /** Returns true if the specified toolchain is a candidate for use with this crosstool. */ + public boolean isCandidateToolchain(CToolchain toolchain) { + return (toolchain.getTargetCpu().equals(getCpu()) + && (getLibc() == null || toolchain.getTargetLibc().equals(getLibc())) + && (getCompiler() == null || toolchain.getCompiler().equals( + getCompiler()))); + } + + @Override + public String toString() { + return describeFlags(); + } + + @Override + public String getCpu() { + return cpu; + } + + @Override + public String getCompiler() { + return compiler; + } + + @Override + public String getLibc() { + return libc; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java new file mode 100644 index 0000000000..a113f5f126 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java @@ -0,0 +1,327 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.io.BaseEncoding; +import com.google.devtools.build.lib.analysis.RedirectChaser; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig; +import com.google.protobuf.TextFormat; +import com.google.protobuf.TextFormat.ParseException; +import com.google.protobuf.UninitializedMessageException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.ExecutionException; + +import javax.annotation.Nullable; + +/** + * A loader that reads Crosstool configuration files and creates CToolchain + * instances from them. + */ +public class CrosstoolConfigurationLoader { + private static final String CROSSTOOL_CONFIGURATION_FILENAME = "CROSSTOOL"; + + /** + * Cache for storing result of toReleaseConfiguration function based on path and md5 sum of + * input file. We can use md5 because result of this function depends only on the file content. + */ + private static final LoadingCache<Pair<Path, String>, CrosstoolConfig.CrosstoolRelease> + crosstoolReleaseCache = CacheBuilder.newBuilder().concurrencyLevel(4).maximumSize(100).build( + new CacheLoader<Pair<Path, String>, CrosstoolConfig.CrosstoolRelease>() { + @Override + public CrosstoolConfig.CrosstoolRelease load(Pair<Path, String> key) throws IOException { + char[] data = FileSystemUtils.readContentAsLatin1(key.first); + return toReleaseConfiguration(key.first.getPathString(), new String(data)); + } + }); + + /** + * A class that holds the results of reading a CROSSTOOL file. + */ + public static class CrosstoolFile { + private final Label crosstoolTop; + private Path crosstoolPath; + private CrosstoolConfig.CrosstoolRelease crosstool; + private String md5; + + CrosstoolFile(Label crosstoolTop) { + this.crosstoolTop = crosstoolTop; + } + + void setCrosstoolPath(Path crosstoolPath) { + this.crosstoolPath = crosstoolPath; + } + + void setCrosstool(CrosstoolConfig.CrosstoolRelease crosstool) { + this.crosstool = crosstool; + } + + void setMd5(String md5) { + this.md5 = md5; + } + + /** + * Returns the crosstool top as resolved. + */ + public Label getCrosstoolTop() { + return crosstoolTop; + } + + /** + * Returns the absolute path from which the CROSSTOOL file was read. + */ + public Path getCrosstoolPath() { + return crosstoolPath; + } + + /** + * Returns the parsed contents of the CROSSTOOL file. + */ + public CrosstoolConfig.CrosstoolRelease getProto() { + return crosstool; + } + + /** + * Returns an MD5 hash of the CROSSTOOL file contents. + */ + public String getMd5() { + return md5; + } + } + + private CrosstoolConfigurationLoader() { + } + + /** + * Reads the given <code>data</code> String, which must be in ascii format, + * into a protocol buffer. It uses the <code>name</code> parameter for error + * messages. + * + * @throws IOException if the parsing failed + */ + @VisibleForTesting + static CrosstoolConfig.CrosstoolRelease toReleaseConfiguration(String name, String data) + throws IOException { + CrosstoolConfig.CrosstoolRelease.Builder builder = + CrosstoolConfig.CrosstoolRelease.newBuilder(); + try { + TextFormat.merge(data, builder); + return builder.build(); + } catch (ParseException e) { + throw new IOException("Could not read the crosstool configuration file '" + name + "', " + + "because of a parser error (" + e.getMessage() + ")"); + } catch (UninitializedMessageException e) { + throw new IOException("Could not read the crosstool configuration file '" + name + "', " + + "because of an incomplete protocol buffer (" + e.getMessage() + ")"); + } + } + + private static boolean findCrosstoolConfiguration( + ConfigurationEnvironment env, + CrosstoolConfigurationLoader.CrosstoolFile file) + throws IOException, InvalidConfigurationException { + Label crosstoolTop = file.getCrosstoolTop(); + Path path = null; + try { + Package containingPackage = env.getTarget(crosstoolTop.getLocalTargetLabel("BUILD")) + .getPackage(); + if (containingPackage == null) { + return false; + } + path = env.getPath(containingPackage, CROSSTOOL_CONFIGURATION_FILENAME); + } catch (SyntaxException e) { + throw new InvalidConfigurationException(e); + } catch (NoSuchThingException e) { + // Handled later + } + + // If we can't find a file, fall back to the provided alternative. + if (path == null || !path.exists()) { + throw new InvalidConfigurationException("The crosstool_top you specified was resolved to '" + + crosstoolTop + "', which does not contain a CROSSTOOL file. " + + "You can use a crosstool from the depot by specifying its label."); + } else { + // Do this before we read the data, so if it changes, we get a different MD5 the next time. + // Alternatively, we could calculate the MD5 of the contents, which we also read, but this + // is faster if the file comes from a file system with md5 support. + file.setCrosstoolPath(path); + String md5 = BaseEncoding.base16().lowerCase().encode(path.getMD5Digest()); + CrosstoolConfig.CrosstoolRelease release; + try { + release = crosstoolReleaseCache.get(new Pair<Path, String>(path, md5)); + file.setCrosstool(release); + file.setMd5(md5); + } catch (ExecutionException e) { + throw new InvalidConfigurationException(e); + } + } + return true; + } + + /** + * Reads a crosstool file. + */ + @Nullable + public static CrosstoolConfigurationLoader.CrosstoolFile readCrosstool( + ConfigurationEnvironment env, Label crosstoolTop) throws InvalidConfigurationException { + crosstoolTop = RedirectChaser.followRedirects(env, crosstoolTop, "crosstool_top"); + if (crosstoolTop == null) { + return null; + } + CrosstoolConfigurationLoader.CrosstoolFile file = + new CrosstoolConfigurationLoader.CrosstoolFile(crosstoolTop); + try { + boolean allDependenciesPresent = findCrosstoolConfiguration(env, file); + return allDependenciesPresent ? file : null; + } catch (IOException e) { + throw new InvalidConfigurationException(e); + } + } + + /** + * Selects a crosstool toolchain corresponding to the given crosstool + * configuration options. If all of these options are null, it returns the default + * toolchain specified in the crosstool release. If only cpu is non-null, it + * returns the default toolchain for that cpu, as specified in the crosstool + * release. Otherwise, all values must be non-null, and this method + * returns the toolchain which matches all of the values. + * + * @throws NullPointerException if {@code release} is null + * @throws InvalidConfigurationException if no matching toolchain can be found, or + * if the input parameters do not obey the constraints described above + */ + public static CrosstoolConfig.CToolchain selectToolchain( + CrosstoolConfig.CrosstoolRelease release, BuildOptions options, + Function<String, String> cpuTransformer) + throws InvalidConfigurationException { + CrosstoolConfigurationIdentifier config = + CrosstoolConfigurationIdentifier.fromReleaseAndCrosstoolConfiguration(release, options); + if ((config.getCompiler() != null) || (config.getLibc() != null)) { + ArrayList<CrosstoolConfig.CToolchain> candidateToolchains = new ArrayList<>(); + for (CrosstoolConfig.CToolchain toolchain : release.getToolchainList()) { + if (config.isCandidateToolchain(toolchain)) { + candidateToolchains.add(toolchain); + } + } + switch (candidateToolchains.size()) { + case 0: { + StringBuilder message = new StringBuilder(); + message.append("No toolchain found for"); + message.append(config.describeFlags()); + message.append(". Valid toolchains are: "); + describeToolchainList(message, release.getToolchainList()); + throw new InvalidConfigurationException(message.toString()); + } + case 1: + return candidateToolchains.get(0); + default: { + StringBuilder message = new StringBuilder(); + message.append("Multiple toolchains found for"); + message.append(config.describeFlags()); + message.append(": "); + describeToolchainList(message, candidateToolchains); + throw new InvalidConfigurationException(message.toString()); + } + } + } + String selectedIdentifier = null; + // We use fake CPU values to allow cross-platform builds for other languages that use the + // C++ toolchain. Translate to the actual target architecture. + String desiredCpu = cpuTransformer.apply(config.getCpu()); + for (CrosstoolConfig.DefaultCpuToolchain selector : release.getDefaultToolchainList()) { + if (selector.getCpu().equals(desiredCpu)) { + selectedIdentifier = selector.getToolchainIdentifier(); + break; + } + } + checkToolChain(selectedIdentifier, desiredCpu); + for (CrosstoolConfig.CToolchain toolchain : release.getToolchainList()) { + if (toolchain.getToolchainIdentifier().equals(selectedIdentifier)) { + return toolchain; + } + } + throw new InvalidConfigurationException("Inconsistent crosstool configuration; no toolchain " + + "corresponding to '" + selectedIdentifier + "' found for cpu '" + config.getCpu() + "'"); + } + + private static String describeToolchainFlags(CrosstoolConfig.CToolchain toolchain) { + return CrosstoolConfigurationIdentifier.fromToolchain(toolchain).describeFlags(); + } + + /** + * Appends a series of toolchain descriptions (as the blaze command line flags + * that would specify that toolchain) to 'message'. + */ + private static void describeToolchainList(StringBuilder message, + Collection<CrosstoolConfig.CToolchain> toolchains) { + message.append("["); + for (CrosstoolConfig.CToolchain toolchain : toolchains) { + message.append(describeToolchainFlags(toolchain)); + message.append(","); + } + message.append("]"); + } + + /** + * Makes sure that {@code selectedIdentifier} is a valid identifier for a toolchain, + * i.e. it starts with a letter or an underscore and continues with only dots, dashes, + * spaces, letters, digits or underscores (i.e. matches the following regular expression: + * "[a-zA-Z_][\.\- \w]*"). + * + * @throws InvalidConfigurationException if selectedIdentifier is null or does not match the + * aforementioned regular expression. + */ + private static void checkToolChain(String selectedIdentifier, String cpu) + throws InvalidConfigurationException { + if (selectedIdentifier == null) { + throw new InvalidConfigurationException("No toolchain found for cpu '" + cpu + "'"); + } + // If you update this regex, please do so in the javadoc comment too, and also in the + // crosstool_config.proto file. + String rx = "[a-zA-Z_][\\.\\- \\w]*"; + if (!selectedIdentifier.matches(rx)) { + throw new InvalidConfigurationException("Toolchain identifier for cpu '" + cpu + "' " + + "is illegal (does not match '" + rx + "')"); + } + } + + public static CrosstoolConfig.CrosstoolRelease getCrosstoolReleaseProto( + ConfigurationEnvironment env, BuildOptions options, + Label crosstoolTop, Function<String, String> cpuTransformer) + throws InvalidConfigurationException { + CrosstoolConfigurationLoader.CrosstoolFile file = + readCrosstool(env, crosstoolTop); + // Make sure that we have the requested toolchain in the result. Throw an exception if not. + selectToolchain(file.getProto(), options, cpuTransformer); + return file.getProto(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationOptions.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationOptions.java new file mode 100644 index 0000000000..e311ab6163 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationOptions.java @@ -0,0 +1,29 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +/** + * A container object which provides crosstool configuration options to the build. + */ +public interface CrosstoolConfigurationOptions { + /** Returns the CPU associated with this crosstool configuration. */ + public String getCpu(); + + /** Returns the compiler associated with this crosstool configuration. */ + public String getCompiler(); + + /** Returns the libc version associated with this crosstool configuration. */ + public String getLibc(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/DiscoveredSourceInputsHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/DiscoveredSourceInputsHelper.java new file mode 100644 index 0000000000..a446125e5c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/DiscoveredSourceInputsHelper.java @@ -0,0 +1,139 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactResolver; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Helper for actions that do include scanning. Currently only deals with source files, so is only + * appropriate for actions that do not discover generated files. Currently does not do .d file + * parsing, so the set of artifacts returned may be an overapproximation to the ones actually used + * during execution. + */ +public class DiscoveredSourceInputsHelper { + + private DiscoveredSourceInputsHelper() { + } + + /** + * Converts PathFragments into source Artifacts using an ArtifactResolver, ignoring any that are + * already in mandatoryInputs. Silently drops any PathFragments that cannot be resolved into + * Artifacts. + */ + public static ImmutableList<Artifact> getDiscoveredInputsFromPaths( + Iterable<Artifact> mandatoryInputs, ArtifactResolver artifactResolver, + Collection<PathFragment> inputPaths) { + Set<PathFragment> knownPathFragments = new HashSet<>(); + for (Artifact input : mandatoryInputs) { + knownPathFragments.add(input.getExecPath()); + } + ImmutableList.Builder<Artifact> foundInputs = ImmutableList.builder(); + for (PathFragment execPath : inputPaths) { + if (!knownPathFragments.add(execPath)) { + // Don't add any inputs that we already added, or original inputs, which we probably + // couldn't convert into artifacts anyway. + continue; + } + Artifact artifact = artifactResolver.resolveSourceArtifact(execPath); + // It is unlikely that this artifact is null, but tolerate the situation just in case. + // It is safe to ignore such paths because dependency checker would identify change in inputs + // (ignored path was used before) and will force action execution. + if (artifact != null) { + foundInputs.add(artifact); + } + } + return foundInputs.build(); + } + + /** + * Converts ActionInputs discovered as inputs during execution into source Artifacts, ignoring any + * that are already in mandatoryInputs or that live in builtInIncludeDirectories. If any + * ActionInputs cannot be resolved, an ActionExecutionException will be thrown. + * + * <p>This method duplicates the functionality of CppCompileAction#populateActionInputs, though it + * is simpler because it need not deal with derived artifacts and doesn't parse the .d file. + */ + public static ImmutableList<Artifact> getDiscoveredInputsFromActionInputs( + Iterable<Artifact> mandatoryInputs, + ArtifactResolver artifactResolver, + Iterable<? extends ActionInput> discoveredInputs, + Iterable<PathFragment> builtInIncludeDirectories, + Action action, + Artifact primaryInput) throws ActionExecutionException { + List<PathFragment> systemIncludePrefixes = new ArrayList<>(); + for (PathFragment includePath : builtInIncludeDirectories) { + if (includePath.isAbsolute()) { + systemIncludePrefixes.add(includePath); + } + } + + // Avoid duplicates by keeping track of the ones we've seen so far, even though duplicates are + // unlikely, since they would have to be inputs to this (non-CppCompile) action and also + // #included by a C++ source file. + Set<Artifact> knownInputs = new HashSet<>(); + Iterables.addAll(knownInputs, mandatoryInputs); + ImmutableList.Builder<Artifact> foundInputs = ImmutableList.builder(); + // Check inclusions. + IncludeProblems problems = new IncludeProblems(); + for (ActionInput input : discoveredInputs) { + if (input instanceof Artifact) { + Artifact artifact = (Artifact) input; + if (knownInputs.add(artifact)) { + foundInputs.add(artifact); + } + continue; + } + PathFragment execPath = new PathFragment(input.getExecPathString()); + if (execPath.isAbsolute()) { + // Absolute includes from system paths are ignored. + if (FileSystemUtils.startsWithAny(execPath, systemIncludePrefixes)) { + continue; + } + // Theoretically, the more sophisticated logic of CppCompileAction#populateActioInputs could + // be used here, to allow absolute includes that started with the execRoot. However, since + // we don't hit this codepath for local execution, that should be unnecessary. If and when + // we examine the results of local execution for scanned includes, that case may need to be + // dealt with. + problems.add(execPath.getPathString()); + } + Artifact artifact = artifactResolver.resolveSourceArtifact(execPath); + if (artifact != null) { + if (knownInputs.add(artifact)) { + foundInputs.add(artifact); + } + } else { + // Abort if we see files that we can't resolve, likely caused by + // undeclared includes or illegal include constructs. + problems.add(execPath.getPathString()); + } + } + problems.assertProblemFree(action, primaryInput); + return foundInputs.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/DwoArtifactsCollector.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/DwoArtifactsCollector.java new file mode 100644 index 0000000000..142a67a98b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/DwoArtifactsCollector.java @@ -0,0 +1,120 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; + +/** + * Provides generic functionality for collecting the .dwo artifacts produced by any target + * that compiles C++ files. Supports both transitive and "only direct outputs" collection. + * Provides accessors for both PIC and non-PIC compilation modes. + */ +public class DwoArtifactsCollector { + + /** + * The .dwo files collected by this target in non-PIC compilation mode (i.e. myobject.dwo). + */ + private final NestedSet<Artifact> dwoArtifacts; + + /** + * The .dwo files collected by this target in PIC compilation mode (i.e. myobject.pic.dwo). + */ + private final NestedSet<Artifact> picDwoArtifacts; + + /** + * Instantiates a "real" collector on meaningful data. + */ + private DwoArtifactsCollector(CcCompilationOutputs compilationOutputs, + Iterable<TransitiveInfoCollection> deps) { + + Preconditions.checkNotNull(compilationOutputs); + Preconditions.checkNotNull(deps); + + // Note: .dwo collection works fine with any order, but tests may assume a + // specific order for readability / simplicity purposes. See + // DebugInfoPackagingTest for details. + NestedSetBuilder<Artifact> dwoBuilder = NestedSetBuilder.compileOrder(); + NestedSetBuilder<Artifact> picDwoBuilder = NestedSetBuilder.compileOrder(); + + dwoBuilder.addAll(compilationOutputs.getDwoFiles()); + picDwoBuilder.addAll(compilationOutputs.getPicDwoFiles()); + + for (TransitiveInfoCollection info : deps) { + CppDebugFileProvider provider = info.getProvider(CppDebugFileProvider.class); + if (provider != null) { + dwoBuilder.addTransitive(provider.getTransitiveDwoFiles()); + picDwoBuilder.addTransitive(provider.getTransitivePicDwoFiles()); + } + } + + dwoArtifacts = dwoBuilder.build(); + picDwoArtifacts = picDwoBuilder.build(); + } + + /** + * Instantiates an empty collector. + */ + private DwoArtifactsCollector() { + dwoArtifacts = NestedSetBuilder.<Artifact>emptySet(Order.COMPILE_ORDER); + picDwoArtifacts = NestedSetBuilder.<Artifact>emptySet(Order.COMPILE_ORDER); + } + + /** + * Returns a new instance that collects direct outputs and transitive dependencies. + * + * @param compilationOutputs the output compilation context for the owning target + * @param deps which of the target's transitive info collections should be visited + */ + public static DwoArtifactsCollector transitiveCollector(CcCompilationOutputs compilationOutputs, + Iterable<TransitiveInfoCollection> deps) { + return new DwoArtifactsCollector(compilationOutputs, deps); + } + + /** + * Returns a new instance that collects direct outputs only. + * + * @param compilationOutputs the output compilation context for the owning target + */ + public static DwoArtifactsCollector directCollector(CcCompilationOutputs compilationOutputs) { + return new DwoArtifactsCollector( + compilationOutputs, ImmutableList.<TransitiveInfoCollection>of()); + } + + /** + * Returns a new instance that doesn't collect anything (its artifact sets are empty). + */ + public static DwoArtifactsCollector emptyCollector() { + return new DwoArtifactsCollector(); + } + + /** + * Returns the .dwo files applicable to non-PIC compilation mode (i.e. myobject.dwo). + */ + public NestedSet<Artifact> getDwoArtifacts() { + return dwoArtifacts; + } + + /** + * Returns the .dwo files applicable to PIC compilation mode (i.e. myobject.pic.dwo). + */ + public NestedSet<Artifact> getPicDwoArtifacts() { + return picDwoArtifacts; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/ExtractInclusionAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/ExtractInclusionAction.java new file mode 100644 index 0000000000..15d7010f7c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/ExtractInclusionAction.java @@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; + +import java.io.IOException; + +/** + * An action which greps for includes over a given .cc or .h file. + * This is a part of the work required for C++ include scanning. + * + * <p>Note that this may run grep-includes over-optimistically, where we previously + * had not. For example, consider a cc_library of generated headers. If another + * library depends on it, and only references one of the headers, the other + * grep-includes will have been wasted. + */ +final class ExtractInclusionAction extends AbstractAction { + + private static final String GUID = "45b43e5a-4734-43bb-a05e-012313808142"; + + /** + * Constructs a new action. + */ + public ExtractInclusionAction(ActionOwner owner, Artifact input, Artifact output) { + super(owner, ImmutableList.of(input), ImmutableList.of(output)); + } + + @Override + protected String computeKey() { + return GUID; + } + + @Override + public String describeStrategy(Executor executor) { + return executor.getContext(CppCompileActionContext.class).strategyLocality(); + } + + @Override + public String getMnemonic() { + return "GrepIncludes"; + } + + @Override + protected String getRawProgressMessage() { + return "Extracting include lines from " + getPrimaryInput().prettyPrint(); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return ResourceSet.ZERO; + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + IncludeScanningContext context = executor.getContext(IncludeScanningContext.class); + try { + context.extractIncludes(actionExecutionContext, this, getPrimaryInput(), + getPrimaryOutput()); + } catch (IOException e) { + throw new ActionExecutionException(e, this, false); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FakeCppCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FakeCppCompileAction.java new file mode 100644 index 0000000000..bd15455491 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FakeCppCompileAction.java @@ -0,0 +1,212 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.UUID; +import java.util.logging.Logger; + +import javax.annotation.Nullable; + +/** + * Action that represents a fake C++ compilation step. + */ +@ThreadCompatible +public class FakeCppCompileAction extends CppCompileAction { + + private static final Logger LOG = Logger.getLogger(FakeCppCompileAction.class.getName()); + + public static final UUID GUID = UUID.fromString("b2d95c91-1434-47ae-a786-816017de8494"); + + private final PathFragment tempOutputFile; + + FakeCppCompileAction(ActionOwner owner, + ImmutableList<String> features, + FeatureConfiguration featureConfiguration, + Artifact sourceFile, + Label sourceLabel, + NestedSet<Artifact> mandatoryInputs, + Artifact outputFile, + PathFragment tempOutputFile, + DotdFile dotdFile, + BuildConfiguration configuration, + CppConfiguration cppConfiguration, + CppCompilationContext context, + ImmutableList<String> copts, + ImmutableList<String> pluginOpts, + Predicate<String> nocopts, + ImmutableList<PathFragment> extraSystemIncludePrefixes, + boolean enableLayeringCheck, + @Nullable String fdoBuildStamp) { + super(owner, features, featureConfiguration, sourceFile, sourceLabel, mandatoryInputs, + outputFile, dotdFile, null, null, null, + configuration, cppConfiguration, + // We only allow inclusion of header files explicitly declared in + // "srcs", so we only use declaredIncludeSrcs, not declaredIncludeDirs. + // (Disallowing use of undeclared headers for cc_fake_binary is needed + // because the header files get included in the runfiles for the + // cc_fake_binary and for the negative compilation tests that depend on + // the cc_fake_binary, and the runfiles must be determined at analysis + // time, so they can't depend on the contents of the ".d" file.) + CppCompilationContext.disallowUndeclaredHeaders(context), null, copts, pluginOpts, nocopts, + extraSystemIncludePrefixes, enableLayeringCheck, fdoBuildStamp, VOID_INCLUDE_RESOLVER, + ImmutableList.<IncludeScannable>of(), + GUID, /*compileHeaderModules=*/false); + this.tempOutputFile = Preconditions.checkNotNull(tempOutputFile); + } + + @Override + @ThreadCompatible + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + + // First, do an normal compilation, to generate the ".d" file. The generated + // object file is built to a temporary location (tempOutputFile) and ignored + // afterwards. + LOG.info("Generating " + getDotdFile()); + CppCompileActionContext context = executor.getContext(CppCompileActionContext.class); + CppCompileActionContext.Reply reply = null; + try { + // We delegate stdout/stderr to nowhere, i.e. same as redirecting to /dev/null. + reply = context.execWithReply( + this, actionExecutionContext.withFileOutErr(new FileOutErr())); + } catch (ExecException e) { + // We ignore failures here (other than capturing the Distributor reply). + // The compilation may well fail (that's the whole point of negative compilation tests). + // We execute it here just for the side effect of generating the ".d" file. + reply = context.getReplyFromException(e, this); + if (reply == null) { + // This can only happen if the ExecException does not come from remote execution. + throw e.toActionExecutionException("", executor.getVerboseFailures(), this); + } + } + IncludeScanningContext scanningContext = executor.getContext(IncludeScanningContext.class); + updateActionInputs(executor.getExecRoot(), scanningContext.getArtifactResolver(), reply); + + // Even cc_fake_binary rules need to properly declare their dependencies... + // In fact, they need to declare their dependencies even more than cc_binary rules do. + // CcCommonConfiguredTarget passes in an empty set of declaredIncludeDirs, + // so this check below will only allow inclusion of header files that are explicitly + // listed in the "srcs" of the cc_fake_binary or in the "srcs" of a cc_library that it + // depends on. + try { + validateInclusions(actionExecutionContext.getMiddlemanExpander(), executor.getEventHandler()); + } catch (ActionExecutionException e) { + // TODO(bazel-team): (2009) make this into an error, once most of the current warnings + // are fixed. + executor.getEventHandler().handle(Event.warn( + getOwner().getLocation(), + e.getMessage() + ";\n this warning may eventually become an error")); + } + + // Generate a fake ".o" file containing the command line needed to generate + // the real object file. + LOG.info("Generating " + outputFile); + + // A cc_fake_binary rule generates fake .o files and a fake target file, + // which merely contain instructions on building the real target. We need to + // be careful to use a new set of output file names in the instructions, as + // to not overwrite the fake output files when someone tries to follow the + // instructions. As the real compilation is executed by the test from its + // runfiles directory (where writing is forbidden), we patch the command + // line to write to $TEST_TMPDIR instead. + final String outputPrefix = "$TEST_TMPDIR/"; + String argv = Joiner.on(' ').join( + Iterables.transform(getArgv(outputFile.getExecPath()), new Function<String, String>() { + @Override + public String apply(String input) { + String result = ShellEscaper.escapeString(input); + if (input.equals(outputFile.getExecPathString()) + || input.equals(getDotdFile().getSafeExecPath().getPathString())) { + result = outputPrefix + result; + } + return result; + } + })); + + // Write the command needed to build the real .o file to the fake .o file. + // Generate a command to ensure that the output directory exists; otherwise + // the compilation would fail. + try { + // Ensure that the .d file and .o file are siblings, so that the "mkdir" below works for + // both. + Preconditions.checkState(outputFile.getExecPath().getParentDirectory().equals( + getDotdFile().getSafeExecPath().getParentDirectory())); + FileSystemUtils.writeContent(outputFile.getPath(), ISO_8859_1, + outputFile.getPath().getBaseName() + ": " + + "mkdir -p " + outputPrefix + "$(dirname " + outputFile.getExecPath() + ")" + + " && " + argv + "\n"); + } catch (IOException e) { + throw new ActionExecutionException("failed to create fake compile command for rule '" + + getOwner().getLabel() + ": " + e.getMessage(), + this, false); + } + } + + @Override + protected PathFragment getInternalOutputFile() { + return tempOutputFile; + } + + @Override + public String getMnemonic() { return "FakeCppCompile"; } + + @Override + public String describeStrategy(Executor executor) { + return "fake"; + } + + @Override + public ResourceSet estimateResourceConsumptionLocal() { + return new ResourceSet(/*memoryMb=*/1, /*cpuUsage=*/0.1, /*ioUsage=*/0.0); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return executor.getContext(CppCompileActionContext.class).estimateResourceConsumption(this); + } + + @Override + protected boolean needsIncludeScanning(Executor executor) { + return false; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoStubAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoStubAction.java new file mode 100644 index 0000000000..f50a1ae5c2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoStubAction.java @@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.vfs.Path; + +/** + * Stub action to be used as the generating action for FDO files that are extracted from the + * FDO zip. + * + * <p>This is needed because the extraction is currently not a bona fide action, therefore, Blaze + * would complain that these files have no generating action if we did not set it to an instance of + * this class. + */ +public class FdoStubAction extends AbstractAction { + public FdoStubAction(ActionOwner owner, Artifact output) { + // TODO(bazel-team): Make extracting the zip file a honest-to-God action so that we can do away + // with this ugliness. + super(owner, ImmutableList.<Artifact>of(), ImmutableList.of(output)); + } + + @Override + public String describeStrategy(Executor executor) { + return ""; + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) { + } + + @Override + public String getMnemonic() { + return "FdoStubAction"; + } + + @Override + protected String computeKey() { + return "fdoStubAction"; + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return ResourceSet.ZERO; + } + + @Override + public void prepare(Path execRoot) { + // The superclass would delete the output files here. We can't let that happen, since this + // action does not in fact create those files; it is only a placeholder and the actual files + // are created *before* the execution phase in FdoSupport.extractFdoZip() + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java new file mode 100644 index 0000000000..911d888952 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java @@ -0,0 +1,679 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.PackageRootResolver; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.skyframe.FileValue; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.lib.vfs.ZipFileSystem; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode; +import com.google.devtools.build.skyframe.SkyFunction; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.regex.Pattern; +import java.util.zip.ZipException; + +/** + * Support class for FDO (feedback directed optimization) and LIPO (lightweight inter-procedural + * optimization). + * + * <p>There is a 1:1 relationship between {@link CppConfiguration} objects and {@code FdoSupport} + * objects. The FDO support of a build configuration can be retrieved using {@link + * CppConfiguration#getFdoSupport()}. + * + * <p>With respect to thread-safety, the {@link #prepareToBuild} method is not thread-safe, and must + * not be called concurrently with other methods on this class. + * + * <p>Here follows a quick run-down of how FDO/LIPO builds work (for non-FDO/LIPO builds, none + * of this applies): + * + * <p>{@link CppConfiguration#prepareHook} is called before the analysis phase, which calls + * {@link #prepareToBuild}, which extracts the FDO .zip (in case we work with an explicitly + * generated FDO profile file) or analyzes the .afdo.imports file next to the .afdo file (if + * AutoFDO is in effect). + * + * <p>.afdo.imports files contain one import a line. A line is two paths separated by a colon, + * with functions in the second path being referenced by functions in the first path. These are + * then put into the imports map. If we do AutoFDO, we don't handle individual .gcda files, so + * gcdaFiles will be empty. + * + * <p>Regular .fdo zip files contain .gcda files (which are added to gcdaFiles) and + * .gcda.imports files. There is one .gcda.imports file for every source file and it contains one + * path in every line, which can either be a path to a source file that contains a function + * referenced by the original source file or the .gcda file for such a referenced file. They + * both are added to the imports map. + * + * <p>If we do LIPO, we create an extra configuration that is called the "LIPO context collector", + * whose job it is to collect information that every configured target compiled with LIPO needs. + * The top-level target of this configuration is the LIPO context (always a cc_binary) and is an + * implicit dependency of every cc_* rule through their :lipo_context_collector attribute. The + * collected information is encapsulated in {@link LipoContextProvider}. + * + * <p>For each C++ compile action in the target configuration, {@link #configureCompilation} is + * called, which adds command line options and input files required for the build. There are + * three cases: + * + * <ul> + * <li>If we do AutoFDO, the .afdo file and the source files containing the functions imported + * by the original source file (as determined from the inputs map) are added. + * <li>If we do FDO, the .gcda file corresponding to the source file is added. + * <li>If we do LIPO, in addition to the .gcda file corresponding to the source file + * (like for FDO) the source files that contain the functions referenced by the source file and + * their .gcda files are added, too. + * </ul> + * + * <p>If we do LIPO, the actual C++ compilation context for LIPO compilation actions is pieced + * together from the CppCompileContext in LipoContextProvider and that of the rule being compiled. + * (see {@link CppCompilationContext#mergeForLipo}) This is so that the include files for the + * extra LIPO sources are found and is, strictly speaking, incorrect, since it also changes the + * declared include directories of the main source file, which in theory can result in the + * compilation passing even though it should fail with undeclared inclusion errors. + * + * <p>During the actual execution of the C++ compile action, the extra sources also need to be + * include scanned, which is the reason why they are {@link IncludeScannable} objects and not + * simple artifacts. We currently create these {@link IncludeScannable} objects by creating actual + * C++ compile actions in the LIPO context collector configuration which are then never executed. + * In fact, these C++ compile actions are never even registered with Skyframe. For this we + * propagate a bit from {@code BuildConfiguration.isActionsEnabled} to + * {@code CachingAnalysisEnvironment.allowRegisteringActions}, which causes actions to be silently + * discarded after configured targets are created. + */ +public class FdoSupport implements Serializable { + + /** + * Path within profile data .zip files that is considered the root of the + * profile information directory tree. + */ + private static final PathFragment ZIP_ROOT = new PathFragment("/"); + + /** + * Returns true if the give fdoFile represents an AutoFdo profile. + */ + public static final boolean isAutoFdo(String fdoFile) { + return CppFileTypes.GCC_AUTO_PROFILE.matches(fdoFile); + } + + /** + * Coverage information output directory passed to {@code --fdo_instrument}, + * or {@code null} if FDO instrumentation is disabled. + */ + private final PathFragment fdoInstrument; + + /** + * Path of the profile file passed to {@code --fdo_optimize}, or + * {@code null} if FDO optimization is disabled. The profile file + * can be a coverage ZIP or an AutoFDO feedback file. + */ + private final Path fdoProfile; + + /** + * Temporary directory to which the coverage ZIP file is extracted to + * (relative to the exec root), or {@code null} if FDO optimization is + * disabled. This is used to create artifacts for the extracted files. + * + * <p>Note that this root is intentionally not registered with the artifact + * factory. + */ + private final Root fdoRoot; + + /** + * The relative path of the FDO root to the exec root. + */ + private final PathFragment fdoRootExecPath; + + /** + * Path of FDO files under the FDO root. + */ + private final PathFragment fdoPath; + + /** + * LIPO mode passed to {@code --lipo}. This is only used if + * {@code fdoProfile != null}. + */ + private final LipoMode lipoMode; + + /** + * Flag indicating whether to use AutoFDO (as opposed to + * instrumentation-based FDO). + */ + private final boolean useAutoFdo; + + /** + * The {@code .gcda} files that have been extracted from the ZIP file, + * relative to the root of the ZIP file. + * + * <p>Set only in {@link #prepareToBuild}. + */ + private ImmutableSet<PathFragment> gcdaFiles = ImmutableSet.of(); + + /** + * Multimap from .gcda file base names to auxiliary input files. + * + * <p>The keys of the multimap are the exec root relative paths of .gcda files + * with the extension removed. The values are the lines from the accompanying + * .gcda.imports file. + * + * <p>The contents of the multimap are copied verbatim from the .gcda.imports + * files and not yet checked for validity. + * + * <p>Set only in {@link #prepareToBuild}. + */ + private ImmutableMultimap<PathFragment, Artifact> imports; + + /** + * Creates an FDO support object. + * + * @param fdoInstrument value of the --fdo_instrument option + * @param fdoProfile path to the profile file passed to --fdo_optimize option + * @param lipoMode value of the --lipo_mode option + */ + public FdoSupport(PathFragment fdoInstrument, Path fdoProfile, LipoMode lipoMode, Path execRoot) { + this.fdoInstrument = fdoInstrument; + this.fdoProfile = fdoProfile; + this.fdoRoot = (fdoProfile == null) + ? null + : Root.asDerivedRoot(execRoot, execRoot.getRelative("blaze-fdo")); + this.fdoRootExecPath = fdoProfile == null + ? null + : fdoRoot.getExecPath().getRelative(new PathFragment("_fdo").getChild( + FileSystemUtils.removeExtension(fdoProfile.getBaseName()))); + this.fdoPath = fdoProfile == null + ? null + : new PathFragment("_fdo").getChild( + FileSystemUtils.removeExtension(fdoProfile.getBaseName())); + this.lipoMode = lipoMode; + this.useAutoFdo = fdoProfile != null && isAutoFdo(fdoProfile.getBaseName()); + } + + public Root getFdoRoot() { + return fdoRoot; + } + + public void declareSkyframeDependencies(SkyFunction.Environment env, Path execRoot) { + if (fdoProfile != null) { + if (isLipoEnabled()) { + // Incrementality is not supported for LIPO builds, see FdoSupport#scannables. + // Ensure that the Skyframe value containing the configuration will not be reused to avoid + // incrementality issues. + PrecomputedValue.dependOnBuildId(env); + return; + } + + // IMPORTANT: Keep the following in sync with #prepareToBuild. + Path path; + if (useAutoFdo) { + path = fdoProfile.getParentDirectory().getRelative( + fdoProfile.getBaseName() + ".imports"); + } else { + path = fdoProfile; + } + env.getValue(FileValue.key(RootedPath.toRootedPathMaybeUnderRoot(path, + ImmutableList.of(execRoot)))); + } + } + + /** + * Prepares the FDO support for building. + * + * <p>When an {@code --fdo_optimize} compile is requested, unpacks the given + * FDO gcda zip file into a clean working directory under execRoot. + * + * @throws FdoException if the FDO ZIP contains a file of unknown type + */ + @ThreadHostile // must be called before starting the build + public void prepareToBuild(Path execRoot, PathFragment genfilesPath, + ArtifactFactory artifactDeserializer, PackageRootResolver resolver) + throws IOException, FdoException { + // The execRoot != null case is only there for testing. We cannot provide a real ZIP file in + // tests because ZipFileSystem does not work with a ZIP on an in-memory file system. + // IMPORTANT: Keep in sync with #declareSkyframeDependencies to avoid incrementality issues. + if (fdoProfile != null && execRoot != null) { + Path fdoDirPath = execRoot.getRelative(fdoRootExecPath); + + FileSystemUtils.deleteTreesBelow(fdoDirPath); + FileSystemUtils.createDirectoryAndParents(fdoDirPath); + + if (useAutoFdo) { + Path fdoImports = fdoProfile.getParentDirectory().getRelative( + fdoProfile.getBaseName() + ".imports"); + if (isLipoEnabled()) { + imports = readAutoFdoImports(artifactDeserializer, fdoImports, genfilesPath, resolver); + } + FileSystemUtils.ensureSymbolicLink( + execRoot.getRelative(getAutoProfilePath()), fdoProfile); + } else { + Path zipFilePath = new ZipFileSystem(fdoProfile).getRootDirectory(); + if (!zipFilePath.getRelative("blaze-out").isDirectory()) { + throw new ZipException("FDO zip files must be zipped directly above 'blaze-out' " + + "for the compiler to find the profile"); + } + ImmutableSet.Builder<PathFragment> gcdaFilesBuilder = ImmutableSet.builder(); + ImmutableMultimap.Builder<PathFragment, Artifact> importsBuilder = + ImmutableMultimap.builder(); + extractFdoZip(artifactDeserializer, zipFilePath, fdoDirPath, + gcdaFilesBuilder, importsBuilder, resolver); + gcdaFiles = gcdaFilesBuilder.build(); + imports = importsBuilder.build(); + } + } + } + + /** + * Recursively extracts a directory from the GCDA ZIP file into a target + * directory. + * + * <p>Imports files are not written to disk. Their content is directly added + * to an internal data structure. + * + * <p>The files are written at $EXECROOT/blaze-fdo/_fdo/(base name of profile zip), and the + * {@code _fdo} directory there is symlinked to from the exec root, so that the file are also + * available at $EXECROOT/_fdo/..., which is their exec path. We need to jump through these + * hoops because the FDO root 1. needs to be a source root, thus the exec path of its root is + * ".", 2. it must not be equal to the exec root so that the artifact factory does not get + * confused, 3. the files under it must be reachable by their exec path from the exec root. + * + * @throws IOException if any of the I/O operations failed + * @throws FdoException if the FDO ZIP contains a file of unknown type + */ + private void extractFdoZip(ArtifactFactory artifactFactory, Path sourceDir, + Path targetDir, ImmutableSet.Builder<PathFragment> gcdaFilesBuilder, + ImmutableMultimap.Builder<PathFragment, Artifact> importsBuilder, + PackageRootResolver resolver) throws IOException, FdoException { + for (Path sourceFile : sourceDir.getDirectoryEntries()) { + Path targetFile = targetDir.getRelative(sourceFile.getBaseName()); + if (sourceFile.isDirectory()) { + targetFile.createDirectory(); + extractFdoZip(artifactFactory, sourceFile, targetFile, gcdaFilesBuilder, importsBuilder, + resolver); + } else { + if (CppFileTypes.COVERAGE_DATA.matches(sourceFile)) { + FileSystemUtils.copyFile(sourceFile, targetFile); + gcdaFilesBuilder.add( + sourceFile.relativeTo(sourceFile.getFileSystem().getRootDirectory())); + } else if (CppFileTypes.COVERAGE_DATA_IMPORTS.matches(sourceFile)) { + readCoverageImports(artifactFactory, sourceFile, importsBuilder, resolver); + } else { + throw new FdoException("FDO ZIP file contained a file of unknown type: " + + sourceFile); + } + } + } + } + + /** + * Reads a .gcda.imports file and stores the imports information. + * + * @throws FdoException if an auxiliary LIPO input was not found + */ + private void readCoverageImports(ArtifactFactory artifactFactory, Path importsFile, + ImmutableMultimap.Builder<PathFragment, Artifact> importsBuilder, + PackageRootResolver resolver) throws IOException, FdoException { + PathFragment key = importsFile.asFragment().relativeTo(ZIP_ROOT); + String baseName = key.getBaseName(); + String ext = Iterables.getOnlyElement(CppFileTypes.COVERAGE_DATA_IMPORTS.getExtensions()); + key = key.replaceName(baseName.substring(0, baseName.length() - ext.length())); + + for (String line : FileSystemUtils.iterateLinesAsLatin1(importsFile)) { + if (!line.isEmpty()) { + // We can't yet fully check the validity of a line. this is done later + // when we actually parse the contained paths. + PathFragment execPath = new PathFragment(line); + if (execPath.isAbsolute()) { + throw new FdoException("Absolute paths not allowed in gcda imports file " + importsFile + + ": " + execPath); + } + Artifact artifact = artifactFactory.deserializeArtifact(new PathFragment(line), resolver); + if (artifact == null) { + throw new FdoException("Auxiliary LIPO input not found: " + line); + } + + importsBuilder.put(key, artifact); + } + } + } + + /** + * Reads a .afdo.imports file and stores the imports information. + */ + private ImmutableMultimap<PathFragment, Artifact> readAutoFdoImports( + ArtifactFactory artifactFactory, Path importsFile, PathFragment genFilePath, + PackageRootResolver resolver) + throws IOException, FdoException { + ImmutableMultimap.Builder<PathFragment, Artifact> importBuilder = ImmutableMultimap.builder(); + for (String line : FileSystemUtils.iterateLinesAsLatin1(importsFile)) { + if (!line.isEmpty()) { + PathFragment key = new PathFragment(line.substring(0, line.indexOf(':'))); + if (key.startsWith(genFilePath)) { + key = key.relativeTo(genFilePath); + } + if (key.isAbsolute()) { + throw new FdoException("Absolute paths not allowed in afdo imports file " + importsFile + + ": " + key); + } + key = FileSystemUtils.replaceSegments(key, "PROTECTED", "_protected", true); + for (String auxFile : line.substring(line.indexOf(':') + 1).split(" ")) { + if (auxFile.length() == 0) { + continue; + } + Artifact artifact = artifactFactory.deserializeArtifact(new PathFragment(auxFile), + resolver); + if (artifact == null) { + throw new FdoException("Auxiliary LIPO input not found: " + auxFile); + } + importBuilder.put(key, artifact); + } + } + } + return importBuilder.build(); + } + + /** + * Returns the imports from the .afdo.imports file of a source file. + * + * @param sourceName the source file + */ + private Collection<Artifact> getAutoFdoImports(PathFragment sourceName) { + Preconditions.checkState(isLipoEnabled()); + ImmutableCollection<Artifact> afdoImports = imports.get(sourceName); + Preconditions.checkState(afdoImports != null, + "AutoFDO import data missing for %s", sourceName); + return afdoImports; + } + + /** + * Returns the imports from the .gcda.imports file of an object file. + * + * @param objDirectory the object directory of the object file's target + * @param objectName the object file + */ + private Iterable<Artifact> getImports(PathFragment objDirectory, PathFragment objectName) { + Preconditions.checkState(isLipoEnabled()); + Preconditions.checkState(imports != null, + "Tried to look up imports of uninitialized FDOSupport"); + PathFragment key = objDirectory.getRelative(FileSystemUtils.removeExtension(objectName)); + ImmutableCollection<Artifact> importsForObject = imports.get(key); + Preconditions.checkState(importsForObject != null, "Import data missing for %s", key); + return importsForObject; + } + + /** + * Configures a compile action builder by adding command line options and + * auxiliary inputs according to the FDO configuration. This method does + * nothing If FDO is disabled. + */ + @ThreadSafe + public void configureCompilation(CppCompileActionBuilder builder, RuleContext ruleContext, + AnalysisEnvironment env, Label lipoLabel, PathFragment sourceName, final Pattern nocopts, + boolean usePic, LipoContextProvider lipoInputProvider) { + // It is a bug if this method is called with useLipo if lipo is disabled. However, it is legal + // if is is called with !useLipo, even though lipo is enabled. + Preconditions.checkArgument(lipoInputProvider == null || isLipoEnabled()); + + // FDO is disabled -> do nothing. + if ((fdoInstrument == null) && (fdoRoot == null)) { + return; + } + + List<String> fdoCopts = new ArrayList<>(); + // Instrumentation phase + if (fdoInstrument != null) { + fdoCopts.add("-fprofile-generate=" + fdoInstrument.getPathString()); + if (lipoMode != LipoMode.OFF) { + fdoCopts.add("-fripa"); + } + } + + // Optimization phase + if (fdoRoot != null) { + // Declare dependency on contents of zip file. + if (env.getSkyframeEnv().valuesMissing()) { + return; + } + Iterable<Artifact> auxiliaryInputs = getAuxiliaryInputs( + ruleContext, env, lipoLabel, sourceName, usePic, lipoInputProvider); + builder.addMandatoryInputs(auxiliaryInputs); + if (!Iterables.isEmpty(auxiliaryInputs)) { + if (useAutoFdo) { + fdoCopts.add("-fauto-profile=" + getAutoProfilePath().getPathString()); + } else { + fdoCopts.add("-fprofile-use=" + fdoRootExecPath); + } + fdoCopts.add("-fprofile-correction"); + if (lipoInputProvider != null) { + fdoCopts.add("-fripa"); + } + } + } + Iterable<String> filteredCopts = fdoCopts; + if (nocopts != null) { + // Filter fdoCopts with nocopts if they exist. + filteredCopts = Iterables.filter(fdoCopts, new Predicate<String>() { + @Override + public boolean apply(String copt) { + return !nocopts.matcher(copt).matches(); + } + }); + } + builder.addCopts(0, filteredCopts); + } + + /** + * Returns the auxiliary files that need to be added to the {@link CppCompileAction}. + */ + private Iterable<Artifact> getAuxiliaryInputs( + RuleContext ruleContext, AnalysisEnvironment env, Label lipoLabel, PathFragment sourceName, + boolean usePic, LipoContextProvider lipoContextProvider) { + // If --fdo_optimize was not specified, we don't have any additional inputs. + if (fdoProfile == null) { + return ImmutableSet.of(); + } else if (useAutoFdo) { + ImmutableSet.Builder<Artifact> auxiliaryInputs = ImmutableSet.builder(); + + Artifact artifact = env.getDerivedArtifact( + fdoPath.getRelative(getAutoProfileRootRelativePath()), fdoRoot); + env.registerAction(new FdoStubAction(ruleContext.getActionOwner(), artifact)); + auxiliaryInputs.add(artifact); + if (lipoContextProvider != null) { + auxiliaryInputs.addAll(getAutoFdoImports(sourceName)); + } + return auxiliaryInputs.build(); + } else { + ImmutableSet.Builder<Artifact> auxiliaryInputs = ImmutableSet.builder(); + + PathFragment objectName = + FileSystemUtils.replaceExtension(sourceName, usePic ? ".pic.o" : ".o"); + + auxiliaryInputs.addAll( + getGcdaArtifactsForObjectFileName(ruleContext, env, objectName, lipoLabel)); + + if (lipoContextProvider != null) { + for (Artifact importedFile : getImports( + getNonLipoObjDir(ruleContext, lipoLabel), objectName)) { + if (CppFileTypes.COVERAGE_DATA.matches(importedFile.getFilename())) { + Artifact gcdaArtifact = getGcdaArtifactsForGcdaPath( + ruleContext, env, importedFile.getExecPath()); + if (gcdaArtifact == null) { + ruleContext.ruleError(String.format( + ".gcda file %s is not in the FDO zip (referenced by source file %s)", + importedFile.getExecPath(), sourceName)); + } else { + auxiliaryInputs.add(gcdaArtifact); + } + } else { + auxiliaryInputs.add(importedFile); + } + } + } + + return auxiliaryInputs.build(); + } + } + + /** + * Returns the .gcda file artifacts for a .gcda path from the .gcda.imports file or null if the + * referenced .gcda file is not in the FDO zip. + */ + private Artifact getGcdaArtifactsForGcdaPath(RuleContext ruleContext, + AnalysisEnvironment env, PathFragment gcdaPath) { + if (!gcdaFiles.contains(gcdaPath)) { + return null; + } + + Artifact artifact = env.getDerivedArtifact(fdoPath.getRelative(gcdaPath), fdoRoot); + env.registerAction(new FdoStubAction(ruleContext.getActionOwner(), artifact)); + return artifact; + } + + private PathFragment getNonLipoObjDir(RuleContext ruleContext, Label label) { + return ruleContext.getConfiguration().getBinFragment() + .getRelative(CppHelper.getObjDirectory(label)); + } + + /** + * Returns a list of .gcda file artifacts for an object file path. + * + * <p>The resulting set is either empty (because no .gcda file exists for the + * given object file) or contains one or two artifacts (the file itself and a + * symlink to it). + */ + private ImmutableList<Artifact> getGcdaArtifactsForObjectFileName(RuleContext ruleContext, + AnalysisEnvironment env, PathFragment objectFileName, Label lipoLabel) { + // We put the .gcda files relative to the location of the .o file in the instrumentation run. + String gcdaExt = Iterables.getOnlyElement(CppFileTypes.COVERAGE_DATA.getExtensions()); + PathFragment baseName = FileSystemUtils.replaceExtension(objectFileName, gcdaExt); + PathFragment gcdaFile = getNonLipoObjDir(ruleContext, lipoLabel).getRelative(baseName); + + if (!gcdaFiles.contains(gcdaFile)) { + // If the object is a .pic.o file and .pic.gcda is not found, we should try finding .gcda too + String picoExt = Iterables.getOnlyElement(CppFileTypes.PIC_OBJECT_FILE.getExtensions()); + baseName = FileSystemUtils.replaceExtension(objectFileName, gcdaExt, picoExt); + if (baseName == null) { + // Object file is not .pic.o + return ImmutableList.of(); + } + gcdaFile = getNonLipoObjDir(ruleContext, lipoLabel).getRelative(baseName); + if (!gcdaFiles.contains(gcdaFile)) { + // .gcda file not found + return ImmutableList.of(); + } + } + + final Artifact artifact = env.getDerivedArtifact(fdoPath.getRelative(gcdaFile), fdoRoot); + env.registerAction(new FdoStubAction(ruleContext.getActionOwner(), artifact)); + + return ImmutableList.of(artifact); + } + + + private PathFragment getAutoProfilePath() { + return fdoRootExecPath.getRelative(getAutoProfileRootRelativePath()); + } + + private PathFragment getAutoProfileRootRelativePath() { + return new PathFragment(fdoProfile.getBaseName()); + } + + /** + * Returns whether LIPO is enabled. + */ + @ThreadSafe + public boolean isLipoEnabled() { + return fdoProfile != null && lipoMode != LipoMode.OFF; + } + + /** + * Returns whether AutoFDO is enabled. + */ + @ThreadSafe + public boolean isAutoFdoEnabled() { + return useAutoFdo; + } + + /** + * Returns an immutable list of command line arguments to add to the linker + * command line. If FDO is disabled, and empty list is returned. + */ + @ThreadSafe + public ImmutableList<String> getLinkOptions() { + return fdoInstrument != null + ? ImmutableList.of("-fprofile-generate=" + fdoInstrument.getPathString()) + : ImmutableList.<String>of(); + } + + /** + * Returns the path of the FDO output tree (relative to the execution root) + * containing the .gcda profile files, or null if FDO is not enabled. + */ + @VisibleForTesting + public PathFragment getFdoOptimizeDir() { + return fdoRootExecPath; + } + + /** + * Returns the path of the FDO zip containing the .gcda profile files, or null + * if FDO is not enabled. + */ + @VisibleForTesting + public Path getFdoOptimizeProfile() { + return fdoProfile; + } + + /** + * Returns the path fragment of the instrumentation output dir for gcc when + * FDO is enabled, or null if FDO is not enabled. + */ + @ThreadSafe + public PathFragment getFdoInstrument() { + return fdoInstrument; + } + + @VisibleForTesting + public void setGcdaFilesForTesting(ImmutableSet<PathFragment> gcdaFiles) { + this.gcdaFiles = gcdaFiles; + } + + /** + * An exception indicating an issue with FDO coverage files. + */ + public static final class FdoException extends Exception { + FdoException(String message) { + super(message); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/HeaderTargetModuleMapProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/HeaderTargetModuleMapProvider.java new file mode 100644 index 0000000000..17e2e5ce11 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/HeaderTargetModuleMapProvider.java @@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.List; + +/** + * A provider for cc_public_library rules to be able to convey the information about the + * header target's module map references to the public library target. + */ +@Immutable +public final class HeaderTargetModuleMapProvider implements TransitiveInfoProvider { + + private final ImmutableList<CppModuleMap> cppModuleMaps; + + public HeaderTargetModuleMapProvider(Iterable<CppModuleMap> cppModuleMaps) { + this.cppModuleMaps = ImmutableList.copyOf(cppModuleMaps); + } + + /** + * Returns the module maps referenced by cc_public_library's headers target. + */ + public List<CppModuleMap> getCppModuleMaps() { + return cppModuleMaps; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/ImplementedCcPublicLibrariesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/ImplementedCcPublicLibrariesProvider.java new file mode 100644 index 0000000000..4f2a5854d9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/ImplementedCcPublicLibrariesProvider.java @@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; + +/** + * A provider for cc_library rules to be able to convey the information about which + * cc_public_library rules they implement to dependent targets. + */ +@Immutable +public final class ImplementedCcPublicLibrariesProvider implements TransitiveInfoProvider { + + private final ImmutableList<Label> implementedCcPublicLibraries; + + public ImplementedCcPublicLibrariesProvider(ImmutableList<Label> implementedCcPublicLibraries) { + this.implementedCcPublicLibraries = implementedCcPublicLibraries; + } + + /** + * Returns the labels for the "$headers" target that are implemented by the target which + * implements this interface. + */ + public ImmutableList<Label> getImplementedCcPublicLibraries() { + return implementedCcPublicLibraries; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeParser.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeParser.java new file mode 100644 index 0000000000..0b60b453ae --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeParser.java @@ -0,0 +1,711 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import com.google.common.io.CharStreams; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.rules.cpp.IncludeParser.Inclusion.Kind; +import com.google.devtools.build.lib.rules.cpp.RemoteIncludeExtractor.RemoteParseData; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import javax.annotation.Nullable; + +/** + * Scans a source file and extracts the literal inclusions it specifies. Does not store results -- + * repeated requests to the same file will result in repeated scans. Clients should implement a + * caching layer in order to avoid unnecessary disk access when requesting an already scanned file. + */ +public class IncludeParser implements SkyValue { + private static final Logger LOG = Logger.getLogger(IncludeParser.class.getName()); + private static final boolean LOG_FINE = LOG.isLoggable(Level.FINE); + private static final boolean LOG_FINER = LOG.isLoggable(Level.FINER); + + /** + * Immutable object representation of the four columns making up a single Rule + * in a Hints set. See {@link Hints} for more details. + */ + private static class Rule { + private enum Type { PATH, FILE, INCLUDE_QUOTE, INCLUDE_ANGLE; } + final Type type; + final Pattern pattern; + final String findRoot; + final Pattern findFilter; + + private Rule(String type, String pattern, String findRoot, Pattern findFilter) { + this.type = Type.valueOf(type.trim().toUpperCase()); + this.pattern = Pattern.compile("^" + pattern + "$"); + this.findRoot = findRoot; + this.findFilter = findFilter; + } + + /** + * @throws PatternSyntaxException, IllegalArgumentException if bad values + * are provided + */ + public Rule(String type, String pattern, String findRoot, String findFilter) { + this(type, pattern, findRoot.replace("\\", "$"), Pattern.compile(findFilter)); + Preconditions.checkArgument((this.type == Type.PATH) || (this.type == Type.FILE)); + } + + public Rule(String type, String pattern, String findRoot) { + this(type, pattern, findRoot, (Pattern) null); + Preconditions.checkArgument((this.type == Type.INCLUDE_QUOTE) + || (this.type == Type.INCLUDE_ANGLE)); + } + + @Override public String toString() { + return "" + type + " " + pattern + " " + findRoot + " " + findFilter; + } + } + + /** + * This class is a representation of the INCLUDE_HINTS file maintained and + * delivered with the remote client. The hints file contains regexp-based rules + * to help this simple include scanner cope with computed includes, which + * would otherwise require a full preprocessor with symbol support. Instead of + * actually processing symbols to evaluate the computed includes, we instead + * apply rules to gather inclusions for matching paths. + * <p> + * The hints file is read, line by line, into a list of rules each of which + * encapsulates a line of four columns. Each non-blank, non-comment line has + * the format: + * + * <pre> + * "file"|"path" match-pattern find-root find-filter + * </pre> + * + * <p> + * The first column specifies whether the line is a rule based on matching + * source <em>files</em> (passed directly to gcc as inputs, or transitively + * #included by other inputs) or include <em>paths</em> (passed to gcc as + * -I, -iquote, or -isystem flags). + * <p> + * The second column is a regexp for files or paths. Whenever a compiler + * argument of the specified type matches that regexp, the rule is taken. (All + * matching rules for every path and file on a compiler command line are + * followed, and the results are combined.) + * <p> + * The third column is a point in the local filesystem from which to extract a + * recursive listing. (This follows symlinks) Backrefs may be used to refer to + * the regexp or its capturing groups. (This is mostly necessary because + * --package_path can cause input paths to carry arbitrary prefixes.) + * <p> + * The fourth column is a regexp applied to each file found by the recursive + * listing. All matching files are treated as dependencies. + */ + public static class Hints implements SkyValue { + + private static final Pattern WS_PAT = Pattern.compile("\\s+"); + + private final Path workingDir; + private final List<Rule> rules = new ArrayList<>(); + private final ArtifactFactory artifactFactory; + + private final LoadingCache<Artifact, Collection<Artifact>> fileLevelHintsCache = + CacheBuilder.newBuilder().build( + new CacheLoader<Artifact, Collection<Artifact>>() { + @Override + public Collection<Artifact> load(Artifact path) { + return getHintedInclusions(Rule.Type.FILE, path.getPath(), path.getRoot()); + } + }); + + private final LoadingCache<Path, Collection<Artifact>> pathLevelHintsCache = + CacheBuilder.newBuilder().build( + new CacheLoader<Path, Collection<Artifact>>() { + @Override + public Collection<Artifact> load(Path path) { + return getHintedInclusions(Rule.Type.PATH, path, null); + } + }); + + /** + * Constructs a hint set for a given working/exec directory and INCLUDE_HINTS file to read. + * + * @param workingDir the working/exec directory that processed paths are relative to + * @param hintsFile the hints file to read + * @throws IOException if the hints file can't be read or parsed + */ + public Hints(Path workingDir, Path hintsFile, ArtifactFactory artifactFactory) + throws IOException { + this.workingDir = workingDir; + this.artifactFactory = artifactFactory; + try (InputStream is = hintsFile.getInputStream()) { + for (String line : CharStreams.readLines(new InputStreamReader(is, "UTF-8"))) { + line = line.trim(); + if (line.length() == 0 || line.startsWith("#")) { + continue; + } + String[] tokens = WS_PAT.split(line); + try { + if (tokens.length == 3) { + rules.add(new Rule(tokens[0], tokens[1], tokens[2])); + } else if (tokens.length == 4) { + rules.add(new Rule(tokens[0], tokens[1], tokens[2], tokens[3])); + } else { + throw new IOException("Malformed hint line: " + line); + } + } catch (PatternSyntaxException e) { + throw new IOException("Malformed hint regex on: " + line + "\n " + e.getMessage()); + } catch (IllegalArgumentException e) { + throw new IOException("Invalid type on: " + line + "\n " + e.getMessage()); + } + } + } + } + + /** + * Returns the "file" type hinted inclusions for a given path, caching results by path. + */ + public Collection<Artifact> getFileLevelHintedInclusions(Artifact path) { + return fileLevelHintsCache.getUnchecked(path); + } + + public Collection<Artifact> getPathLevelHintedInclusions(Path path) { + return pathLevelHintsCache.getUnchecked(path); + } + + /** + * Performs the work of matching a given file/path of a specified file/path type against the + * hints and returns the expanded paths. + */ + private Collection<Artifact> getHintedInclusions(Rule.Type type, Path path, + @Nullable Root sourceRoot) { + String pathString = path.getPathString(); + // Delay creation until we know we need one. Use a TreeSet to make sure that the results are + // sorted with a stable order and unique. + Set<Path> hints = null; + for (final Rule rule : rules) { + if (type != rule.type) { + continue; + } + Matcher m = rule.pattern.matcher(pathString); + if (!m.matches()) { + continue; + } + if (hints == null) { hints = Sets.newTreeSet(); } + Path root = workingDir.getRelative(m.replaceFirst(rule.findRoot)); + if (LOG_FINE) { + LOG.fine("hint for " + rule.type + " " + pathString + " root: " + root); + } + try { + // The assumption is made here that all files specified by this hint are under the same + // package path as the original file -- this filesystem tree traversal is completely + // ignorant of package paths. This could be violated if there were a hint that resolved to + // foo/**/*.h, there was a package foo/bar, and the packages foo and foo/bar were in + // different package paths. In that case, this traversal would fail to pick up + // foo/bar/**/*.h. No examples of this currently exist in the INCLUDE_HINTS + // file. + FileSystemUtils.traverseTree(hints, root, new Predicate<Path>() { + @Override + public boolean apply(Path p) { + boolean take = p.isFile() && rule.findFilter.matcher(p.getPathString()).matches(); + if (LOG_FINER && take) { + LOG.finer("hinted include: " + p); + } + return take; + } + }); + } catch (IOException e) { + LOG.warning("Error in hint expansion: " + e); + } + } + if (hints != null && !hints.isEmpty()) { + // Transform paths into source artifacts (all hints must be to source artifacts). + List<Artifact> result = new ArrayList<>(hints.size()); + for (Path hint : hints) { + if (hint.startsWith(workingDir)) { + // Paths that are under the execRoot can be resolved as source artifacts as usual. All + // include directories are specified relative to the execRoot, and so fall here. + result.add(Preconditions.checkNotNull( + artifactFactory.resolveSourceArtifact(hint.relativeTo(workingDir)), hint)); + } else { + // The file passed in might not have been under the execRoot, for instance + // <workspace>/foo/foo.cc. + Preconditions.checkNotNull(sourceRoot, "%s %s", path, hint); + Path sourcePath = sourceRoot.getPath(); + Preconditions.checkState(hint.startsWith(sourcePath), + "%s %s %s", hint, path, sourceRoot); + result.add(Preconditions.checkNotNull( + artifactFactory.getSourceArtifact(hint.relativeTo(sourcePath), sourceRoot))); + } + } + return result; + } else { + return ImmutableList.of(); + } + } + + private Collection<Inclusion> getHintedInclusions(Artifact path) { + String pathString = path.getPath().getPathString(); + // Delay creation until we know we need one. Use a LinkedHashSet to make sure that the results + // are sorted with a stable order and unique. + Set<Inclusion> hints = null; + for (final Rule rule : rules) { + if ((rule.type != Rule.Type.INCLUDE_ANGLE) && (rule.type != Rule.Type.INCLUDE_QUOTE)) { + continue; + } + Matcher m = rule.pattern.matcher(pathString); + if (!m.matches()) { + continue; + } + if (hints == null) { hints = Sets.newLinkedHashSet(); } + Inclusion inclusion = new Inclusion(rule.findRoot, rule.type == Rule.Type.INCLUDE_QUOTE + ? Kind.QUOTE : Kind.ANGLE); + hints.add(inclusion); + if (LOG_FINE) { + LOG.fine("hint for " + rule.type + " " + pathString + " root: " + inclusion); + } + } + if (hints != null && !hints.isEmpty()) { + return ImmutableList.copyOf(hints); + } else { + return ImmutableList.of(); + } + } + } + + public Hints getHints() { + return hints; + } + + /** + * An immutable inclusion tuple. This models an {@code #include} or {@code + * #include_next} line in a file without the context how this file got + * included. + */ + public static class Inclusion { + /** The format of the #include in the source file -- quoted, angle bracket, etc. */ + public enum Kind { + /** Quote includes: {@code #include "name"}. */ + QUOTE, + + /** Angle bracket includes: {@code #include <name>}. */ + ANGLE, + + /** Quote next includes: {@code #include_next "name"}. */ + NEXT_QUOTE, + + /** Angle next includes: {@code #include_next <name>}. */ + NEXT_ANGLE, + + /** Computed or other unhandlable includes: {@code #include HEADER}. */ + OTHER; + + /** + * Returns true if this is an {@code #include_next} inclusion, + */ + public boolean isNext() { + return this == NEXT_ANGLE || this == NEXT_QUOTE; + } + } + + /** The kind of inclusion. */ + public final Kind kind; + /** The relative path of the inclusion. */ + public final PathFragment pathFragment; + + public Inclusion(String includeTarget, Kind kind) { + this.kind = kind; + this.pathFragment = new PathFragment(includeTarget); + } + + public Inclusion(PathFragment pathFragment, Kind kind) { + this.kind = kind; + this.pathFragment = Preconditions.checkNotNull(pathFragment); + } + + public String getPathString() { + return pathFragment.getPathString(); + } + + @Override + public String toString() { + return kind.toString() + ":" + pathFragment.getPathString(); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof Inclusion)) { + return false; + } + Inclusion that = (Inclusion) o; + return kind == that.kind && pathFragment.equals(that.pathFragment); + } + + @Override + public int hashCode() { + return pathFragment.hashCode() * 37 + kind.hashCode(); + } + } + + /** + * The externally-scoped immutable hints helper that is shared by all scanners. + */ + private final Hints hints; + + /** + * A scanner that extracts includes from an individual files remotely, used when scanning files + * generated remotely. + */ + private final Supplier<? extends RemoteIncludeExtractor> remoteExtractor; + + /** + * Constructs a new FileParser. + * @param remoteExtractor a processor that extracts includes from an individual file remotely. + * @param hints regexps for converting computed includes into simple strings + */ + public IncludeParser(@Nullable RemoteIncludeExtractor remoteExtractor, Hints hints) { + this.hints = hints; + this.remoteExtractor = Suppliers.ofInstance(remoteExtractor); + } + + /** + * Constructs a new FileParser. + * @param remoteExtractorSupplier a supplier of a processor that extracts includes from an + * individual file remotely. + * @param hints regexps for converting computed includes into simple strings + */ + public IncludeParser(Supplier<? extends RemoteIncludeExtractor> remoteExtractorSupplier, + Hints hints) { + this.hints = hints; + this.remoteExtractor = remoteExtractorSupplier; + } + + /** + * Skips whitespace, \+NL pairs, and block-style / * * / comments. Assumes + * line comments are handled outside. Does not handle digraphs, trigraphs or + * decahexagraphs. + * + * @param chars characters to scan + * @param pos the starting position + * @return the resulting position after skipping whitespace and comments. + */ + protected static int skipWhitespace(char[] chars, int pos, int end) { + while (pos < end) { + if (Character.isWhitespace(chars[pos])) { + pos++; + } else if (chars[pos] == '\\' && pos + 1 < end && chars[pos + 1] == '\n') { + pos++; + } else if (chars[pos] == '/' && pos + 1 < end && chars[pos + 1] == '*') { + pos += 2; + while (pos < end - 1) { + if (chars[pos++] == '*') { + if (chars[pos] == '/') { + pos++; + break; // proper comment end + } + } + } + } else { // not whitespace + return pos; + } + } + return pos; // pos == len, meaning we fell off the end. + } + + /** + * Checks for and skips a given token. + * + * @param chars characters to scan + * @param pos the starting position + * @param expected the expected token + * @return the resulting position if found, otherwise -1 + */ + protected static int expect(char[] chars, int pos, int end, String expected) { + int si = 0; + int expectedLen = expected.length(); + while (pos < end) { + if (si == expectedLen) { + return pos; + } + if (chars[pos++] != expected.charAt(si++)) { + return -1; + } + } + return -1; + } + + /** + * Finds the index of a given character token from a starting pos. + * + * @param chars characters to scan + * @param pos the starting position + * @param echar the character to find + * @return the resulting position of echar if found, otherwise -1 + */ + private static int indexOf(char[] chars, int pos, int end, char echar) { + while (pos < end) { + if (chars[pos] == echar) { + return pos; + } + pos++; + } + return -1; + } + + private static final Pattern BS_NL_PAT = Pattern.compile("\\\\" + "\n"); + + // Keep this in sync with the auxiliary binary's scanning output format. + private static final ImmutableMap<Character, Kind> KIND_MAP = ImmutableMap.of( + '"', Kind.QUOTE, + '<', Kind.ANGLE, + 'q', Kind.NEXT_QUOTE, + 'a', Kind.NEXT_ANGLE); + + /** + * Processes the output generated by an auxiliary include-scanning binary. Closes the stream upon + * completion. + * + * <p>If a source file has the following include statements: + * <pre> + * #include <string> + * #include "directory/header.h" + * </pre> + * + * <p>Then the output file has the following contents: + * <pre> + * "directory/header.h + * <string + * </pre> + * <p>Each line of the output is translated into an Inclusion object. + */ + public static List<Inclusion> processIncludes(Object streamName, InputStream is) + throws IOException { + List<Inclusion> inclusions = new ArrayList<>(); + InputStreamReader reader = new InputStreamReader(is, ISO_8859_1); + try { + for (String line : CharStreams.readLines(reader)) { + char qchar = line.charAt(0); + String name = line.substring(1); + Inclusion.Kind kind = KIND_MAP.get(qchar); + if (kind == null) { + throw new IOException("Illegal inclusion kind '" + qchar + "'"); + } + inclusions.add(new Inclusion(name, kind)); + } + } catch (IOException e) { + throw new IOException("Error reading include file " + streamName + ": " + e.getMessage()); + } finally { + reader.close(); + } + return inclusions; + } + + @VisibleForTesting + Inclusion extractInclusion(String line) { + return extractInclusion(line.toCharArray(), 0, line.length()); + } + + /** + * Extracts a new, unresolved an Inclusion from a line of source. + * + * @param chars the char array containing the line chars to parse + * @param lineBegin the position of the first character in the line + * @param lineEnd the position of the character after the last + * @return the inclusion object if possible, null if none + */ + private Inclusion extractInclusion(char[] chars, int lineBegin, int lineEnd) { + // expect WS#WS(include|include_next)WS("name"|<name>|junk) + int pos = expectIncludeKeyword(chars, lineBegin, lineEnd); + if (pos == -1 || pos == lineEnd) { + return null; + } + boolean isNext = false; + int npos = expect(chars, pos, lineEnd, "_next"); + if (npos >= 0) { + isNext = true; + pos = npos; + } + if ((pos = skipWhitespace(chars, pos, lineEnd)) == lineEnd) { + return null; + } + if (chars[pos] == '"' || chars[pos] == '<') { + char qchar = chars[pos++]; + int spos = pos; + pos = indexOf(chars, pos + 1, lineEnd, qchar == '<' ? '>' : '"'); + if (pos < 0) { + return null; + } + if (chars[spos] == '/') { + return null; // disallow absolute paths + } + String name = new String(chars, spos, pos - spos); + if (name.contains("\n")) { // strip any \+NL pairs within name + name = BS_NL_PAT.matcher(name).replaceAll(""); + } + if (isNext) { + return new Inclusion(name, qchar == '"' ? Kind.NEXT_QUOTE : Kind.NEXT_ANGLE); + } else { + return new Inclusion(name, qchar == '"' ? Kind.QUOTE : Kind.ANGLE); + } + } else { + return createOtherInclusion(new String(chars, pos, lineEnd - pos)); + } + } + + /** + * Extracts all inclusions from characters of a file. + * + * @param chars the file contents to parse & extract inclusions from + * @return a new set of inclusions, normalized to the cache + */ + @VisibleForTesting + List<Inclusion> extractInclusions(char[] chars) { + List<Inclusion> inclusions = new ArrayList<>(); + int lineBegin = 0; // the first char of each line + int end = chars.length; // the file end + while (lineBegin < end) { + int lineEnd = lineBegin; // the char after the last non-\n in each line + // skip to the next \n or after end of buffer, ignoring continuations + while (lineEnd < end) { + if (chars[lineEnd] == '\n') { + break; + } else if (chars[lineEnd] == '\\') { + lineEnd++; + if (chars[lineEnd] == '\n') { + lineEnd++; + } + } else { + lineEnd++; + } + } + + // TODO(bazel-team) handle multiline block comments /* */ for the cases: + // /* blah blah blah + // lalala */ #include "foo.h" + // and: + // /* blah + // #include "foo.h" + // */ + + // extract the inclusion, and save only the kind we care about. + Inclusion inclusion = extractInclusion(chars, lineBegin, lineEnd); + if (inclusion != null) { + if (isValidInclusionKind(inclusion.kind)) { + inclusions.add(inclusion); + } else { + //System.err.println("Funky include " + inclusion + " in " + file); + } + } + lineBegin = lineEnd + 1; // next line starts after the previous line + } + return inclusions; + } + + /** + * Extracts all inclusions from a given source file. + * + * @param file the file to parse & extract inclusions from + * @param greppedFile if non-null, this file has the already-grepped include lines of file. + * @param actionExecutionContext Services in the scope of the action, like the stream to which + * scanning messages are printed + * @return a new set of inclusions, normalized to the cache + */ + public Collection<Inclusion> extractInclusions(Artifact file, @Nullable Path greppedFile, + ActionExecutionContext actionExecutionContext) + throws IOException, InterruptedException { + Collection<Inclusion> inclusions; + if (greppedFile != null) { + inclusions = processIncludes(greppedFile, greppedFile.getInputStream()); + } else { + RemoteParseData remoteParseData = remoteExtractor.get() == null + ? null + : remoteExtractor.get().shouldParseRemotely(file.getPath()); + if (remoteParseData != null && remoteParseData.shouldParseRemotely()) { + inclusions = + remoteExtractor.get().extractInclusions(file, actionExecutionContext, + remoteParseData); + } else { + inclusions = extractInclusions(FileSystemUtils.readContentAsLatin1(file.getPath())); + } + } + if (hints != null) { + inclusions.addAll(hints.getHintedInclusions(file)); + } + return ImmutableList.copyOf(inclusions); + } + + /** + * Parses include keyword in the provided char array and returns position + * immediately after include keyword or -1 if keyword was not found. Can be + * overridden by subclasses. + */ + protected int expectIncludeKeyword(char[] chars, int position, int end) { + int pos = expect(chars, skipWhitespace(chars, position, end), end, "#"); + if (pos > 0) { + int npos = skipWhitespace(chars, pos, end); + if ((pos = expect(chars, npos, end, "include")) > 0) { + return pos; + } else if ((pos = expect(chars, npos, end, "import")) > 0) { + if (expect(chars, pos, end, "_") == -1) { // Needed to avoid #import_next. + return pos; + } + } + } + return -1; + } + + /** + * Returns true if we interested in the given inclusion kind. Can be + * overridden by the subclass. + */ + protected boolean isValidInclusionKind(Kind kind) { + return kind != Kind.OTHER; + } + + /** + * Returns inclusion object for non-standard inclusion cases or null if + * inclusion should be ignored. + */ + protected Inclusion createOtherInclusion(String inclusionContent) { + return new Inclusion(inclusionContent, Kind.OTHER); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeProblems.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeProblems.java new file mode 100644 index 0000000000..f6be87747e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeProblems.java @@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.Artifact; + +/** + * Accumulator for problems encountered while reading or validating inclusion + * results. + */ +class IncludeProblems { + + private StringBuilder message; // null when no problems + + void add(String included) { + if (message == null) { message = new StringBuilder(); } + message.append("\n '" + included + "'"); + } + + boolean hasProblems() { return message != null; } + + String getMessage(Action action, Artifact sourceFile) { + if (message != null) { + return "undeclared inclusion(s) in rule '" + action.getOwner().getLabel() + "':\n" + + "this rule is missing dependency declarations for the following files " + + "included by '" + sourceFile.prettyPrint() + "':" + + message; + } + return null; + } + + void assertProblemFree(Action action, Artifact sourceFile) throws ActionExecutionException { + if (hasProblems()) { + throw new ActionExecutionException(getMessage(action, sourceFile), action, false); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScannable.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScannable.java new file mode 100644 index 0000000000..9c70090ac0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScannable.java @@ -0,0 +1,90 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * To be implemented by actions (such as C++ compilation steps) whose inputs + * can be scanned to discover other implicit inputs (such as C++ header files). + * + * <p>This is useful for remote execution strategies to be able to compute the + * complete set of files that must be distributed in order to execute such an action. + */ +public interface IncludeScannable { + + /** + * Returns the built-in list of system include paths for the toolchain compiler. All paths in this + * list should be relative to the exec directory. They may be absolute if they are also installed + * on the remote build nodes or for local compilation. + */ + List<PathFragment> getBuiltInIncludeDirectories(); + + /** + * Returns an immutable list of "-iquote" include paths that should be used by + * the IncludeScanner for this action. GCC searches these paths first, but + * only for {@code #include "foo"}, not for {@code #include <foo>}. + */ + List<PathFragment> getQuoteIncludeDirs(); + + /** + * Returns an immutable list of "-I" include paths that should be used by the + * IncludeScanner for this action. GCC searches these paths ahead of the + * system include paths, but after "-iquote" include paths. + */ + List<PathFragment> getIncludeDirs(); + + /** + * Returns an immutable list of "-isystem" include paths that should be used + * by the IncludeScanner for this action. GCC searches these paths ahead of + * the built-in system include paths, but after all other paths. "-isystem" + * paths are treated the same as normal system directories. + */ + List<PathFragment> getSystemIncludeDirs(); + + /** + * Returns an immutable list of "-include" inclusions specified explicitly on + * the command line of this action. GCC will imagine that these files have + * been quote-included at the beginning of each source file. + */ + List<String> getCmdlineIncludes(); + + /** + * Returns an immutable list of sources that the IncludeScanner should scan + * for this action. + */ + Collection<Artifact> getIncludeScannerSources(); + + /** + * Returns additional scannables that need also be scanned when scanning this + * scannable. May be empty but not null. This is not evaluated recursively. + */ + Iterable<IncludeScannable> getAuxiliaryScannables(); + + /** + * Returns a map of generated files:files grepped for headers which may be reached during include + * scanning. Generated files which are reached, but not in the key set, must be ignored. + * + * <p>If grepping of output files is not enabled via --extract_generated_inclusions, keys + * should just map to null. + */ + Map<Artifact, Path> getLegalGeneratedScannerFileMap(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanner.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanner.java new file mode 100644 index 0000000000..9c00efd64a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanner.java @@ -0,0 +1,177 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.UserExecException; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Scans source files to determine the bounding set of transitively referenced include files. + * + * <p>Note that include scanning is performance-critical code. + */ +public interface IncludeScanner { + /** + * Processes a source file and a list of includes extracted from command line + * flags. Adds all found files to the provided set {@code includes}. This + * method takes into account the path- and file-level hints that are part of + * this include scanner. + */ + public void process(Artifact source, Map<Artifact, Path> legalOutputPaths, + List<String> cmdlineIncludes, Set<Artifact> includes, + ActionExecutionContext actionExecutionContext) + throws IOException, ExecException, InterruptedException; + + /** Supplies IncludeScanners upon request. */ + interface IncludeScannerSupplier { + /** Returns the possibly shared scanner to be used for a given pair of include paths. */ + IncludeScanner scannerFor(List<Path> quoteIncludePaths, List<Path> includePaths); + } + + /** + * Helper class that exists just to provide a static method that prepares the arguments with which + * to call an IncludeScanner. + */ + class IncludeScanningPreparer { + private IncludeScanningPreparer() {} + + /** + * Returns the files transitively included by the source files of the given IncludeScannable. + * + * @param action IncludeScannable whose sources' transitive includes will be returned. + * @param includeScannerSupplier supplies IncludeScanners to actually do the transitive + * scanning (and caching results) for a given source file. + * @param actionExecutionContext the context for {@code action}. + * @param profilerTaskName what the {@link Profiler} should record this call for. + */ + public static Collection<Artifact> scanForIncludedInputs(IncludeScannable action, + IncludeScannerSupplier includeScannerSupplier, + ActionExecutionContext actionExecutionContext, + String profilerTaskName) + throws ExecException, InterruptedException, ActionExecutionException { + + Set<Artifact> includes = Sets.newConcurrentHashSet(); + + Executor executor = actionExecutionContext.getExecutor(); + Path execRoot = executor.getExecRoot(); + + final List<Path> absoluteBuiltInIncludeDirs = new ArrayList<>(); + + Profiler profiler = Profiler.instance(); + try { + profiler.startTask(ProfilerTask.SCANNER, profilerTaskName); + + // We need to scan the action itself, but also the auxiliary scannables + // (for LIPO). There is no need to call getAuxiliaryScannables + // recursively. + for (IncludeScannable scannable : + Iterables.concat(ImmutableList.of(action), action.getAuxiliaryScannables())) { + + Map<Artifact, Path> legalOutputPaths = scannable.getLegalGeneratedScannerFileMap(); + List<PathFragment> includeDirs = new ArrayList<>(scannable.getIncludeDirs()); + List<PathFragment> quoteIncludeDirs = scannable.getQuoteIncludeDirs(); + List<String> cmdlineIncludes = scannable.getCmdlineIncludes(); + + for (PathFragment pathFragment : scannable.getSystemIncludeDirs()) { + includeDirs.add(pathFragment); + } + + // Add the system include paths to the list of include paths. + for (PathFragment pathFragment : action.getBuiltInIncludeDirectories()) { + if (pathFragment.isAbsolute()) { + absoluteBuiltInIncludeDirs.add(execRoot.getRelative(pathFragment)); + } + includeDirs.add(pathFragment); + } + + IncludeScanner scanner = includeScannerSupplier.scannerFor( + relativeTo(execRoot, quoteIncludeDirs), + relativeTo(execRoot, includeDirs)); + + for (Artifact source : scannable.getIncludeScannerSources()) { + // Add all include scanning entry points to the inputs; this is necessary + // when we have more than one source to scan from, for example when building + // C++ modules. + // In that case we have one of two cases: + // 1. We compile a header module - there, the .cppmap file is the main source file + // (which we do not include-scan, as that would require an extra parser), and + // thus already in the input; all headers in the .cppmap file are our entry points + // for include scanning, but are not yet in the inputs - they get added here. + // 2. We compile an object file that uses a header module; currently using a header + // module requires all headers it can reference to be available for the compilation. + // The header module can reference headers that are not in the transitive include + // closure of the current translation unit. Therefore, {@code CppCompileAction} + // adds all headers specified transitively for compiled header modules as include + // scanning entry points, and we need to add the entry points to the inputs here. + includes.add(source); + scanner.process(source, legalOutputPaths, cmdlineIncludes, includes, + actionExecutionContext); + } + } + } catch (IOException e) { + throw new EnvironmentalExecException(e.getMessage()); + } finally { + profiler.completeTask(ProfilerTask.SCANNER); + } + + // Collect inputs and output + List<Artifact> inputs = new ArrayList<>(); + IncludeProblems includeProblems = new IncludeProblems(); + for (Artifact included : includes) { + if (FileSystemUtils.startsWithAny(included.getPath(), absoluteBuiltInIncludeDirs)) { + // Skip include files found in absolute include directories. This currently only applies + // to grte. + continue; + } + if (included.getRoot().getPath().getParentDirectory() == null) { + throw new UserExecException( + "illegal absolute path to include file: " + included.getPath()); + } + inputs.add(included); + } + return inputs; + } + + private static List<Path> relativeTo( + Path path, Collection<PathFragment> fragments) { + List<Path> result = Lists.newArrayListWithCapacity(fragments.size()); + for (PathFragment fragment : fragments) { + result.add(path.getRelative(fragment)); + } + return result; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanningContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanningContext.java new file mode 100644 index 0000000000..69cd26b6bd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanningContext.java @@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionMetadata; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactResolver; +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +import java.io.IOException; + +/** + * Context for actions that do include scanning. + */ +public interface IncludeScanningContext extends ActionContext { + /** + * Extracts the set of include files from a source file. + * + * @param actionExecutionContext the execution context + * @param resourceOwner the resource owner + * @param primaryInput the source file to be include scanned + * @param primaryOutput the output file where the results should be put + */ + void extractIncludes(ActionExecutionContext actionExecutionContext, + ActionMetadata resourceOwner, Artifact primaryInput, Artifact primaryOutput) + throws IOException, InterruptedException; + + /** + * Returns the artifact resolver. + */ + ArtifactResolver getArtifactResolver(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/Link.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/Link.java new file mode 100644 index 0000000000..26175ebb95 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/Link.java @@ -0,0 +1,274 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.util.FileTypeSet; + +import java.util.Iterator; + +/** + * Utility types and methods for generating command lines for the linker, given + * a CppLinkAction or LinkConfiguration. + * + * <p>The linker commands, e.g. "ar", may not be functional, i.e. + * they may mutate the output file rather than overwriting it. + * To avoid this, we need to delete the output file before invoking the + * command. But that is not done by this class; deleting the output + * file is the responsibility of the classes derived from LinkStrategy. + */ +public abstract class Link { + + private Link() {} // uninstantiable + + /** The set of valid linker input files. */ + public static final FileTypeSet VALID_LINKER_INPUTS = FileTypeSet.of( + CppFileTypes.ARCHIVE, CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_LIBRARY, CppFileTypes.ALWAYS_LINK_PIC_LIBRARY, + CppFileTypes.OBJECT_FILE, CppFileTypes.PIC_OBJECT_FILE, + CppFileTypes.SHARED_LIBRARY, CppFileTypes.VERSIONED_SHARED_LIBRARY, + CppFileTypes.INTERFACE_SHARED_LIBRARY); + + /** + * These file are supposed to be added using {@code addLibrary()} calls to {@link CppLinkAction} + * but will never be expanded to their constituent {@code .o} files. {@link CppLinkAction} checks + * that these files are never added as non-libraries. + */ + public static final FileTypeSet SHARED_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY, + CppFileTypes.INTERFACE_SHARED_LIBRARY); + + /** + * These need special handling when --thin_archive is true. {@link CppLinkAction} checks that + * these files are never added as non-libraries. + */ + public static final FileTypeSet ARCHIVE_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.ARCHIVE, + CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_LIBRARY, + CppFileTypes.ALWAYS_LINK_PIC_LIBRARY); + + public static final FileTypeSet ARCHIVE_FILETYPES = FileTypeSet.of( + CppFileTypes.ARCHIVE, + CppFileTypes.PIC_ARCHIVE); + + public static final FileTypeSet LINK_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.ALWAYS_LINK_LIBRARY, + CppFileTypes.ALWAYS_LINK_PIC_LIBRARY); + + + /** The set of object files */ + public static final FileTypeSet OBJECT_FILETYPES = FileTypeSet.of( + CppFileTypes.OBJECT_FILE, + CppFileTypes.PIC_OBJECT_FILE); + + /** + * Prefix that is prepended to command line entries that refer to the output + * of cc_fake_binary compile actions. This is a bad hack to signal to the code + * in {@code CppLinkAction#executeFake(Executor, FileOutErr)} that it needs + * special handling. + */ + public static final String FAKE_OBJECT_PREFIX = "fake:"; + + /** + * Types of ELF files that can be created by the linker (.a, .so, .lo, + * executable). + */ + public enum LinkTargetType { + /** A normal static archive. */ + STATIC_LIBRARY(".a", true), + + /** A static archive with .pic.o object files (compiled with -fPIC). */ + PIC_STATIC_LIBRARY(".pic.a", true), + + /** An interface dynamic library. */ + INTERFACE_DYNAMIC_LIBRARY(".ifso", false), + + /** A dynamic library. */ + DYNAMIC_LIBRARY(".so", false), + + /** A static archive without removal of unused object files. */ + ALWAYS_LINK_STATIC_LIBRARY(".lo", true), + + /** A PIC static archive without removal of unused object files. */ + ALWAYS_LINK_PIC_STATIC_LIBRARY(".pic.lo", true), + + /** An executable binary. */ + EXECUTABLE("", false); + + private final String extension; + private final boolean staticLibraryLink; + + private LinkTargetType(String extension, boolean staticLibraryLink) { + this.extension = extension; + this.staticLibraryLink = staticLibraryLink; + } + + public String getExtension() { + return extension; + } + + public boolean isStaticLibraryLink() { + return staticLibraryLink; + } + } + + /** + * The degree of "staticness" of symbol resolution during linking. + */ + public enum LinkStaticness { + FULLY_STATIC, // Static binding of all symbols. + MOSTLY_STATIC, // Use dynamic binding only for symbols from glibc. + DYNAMIC, // Use dynamic binding wherever possible. + } + + /** + * Types of archive. + */ + public enum ArchiveType { + FAT, // Regular archive that includes its members. + THIN, // Thin archive that just points to its members. + START_END_LIB // A --start-lib ... --end-lib group in the command line. + } + + static boolean useStartEndLib(LinkerInput linkerInput, ArchiveType archiveType) { + // TODO(bazel-team): Figure out if PicArchives are actually used. For it to be used, both + // linkingStatically and linkShared must me true, we must be in opt mode and cpu has to be k8. + return archiveType == ArchiveType.START_END_LIB + && ARCHIVE_FILETYPES.matches(linkerInput.getArtifact().getFilename()) + && linkerInput.containsObjectFiles(); + } + + /** + * Replace always used archives with its members. This is used to build the linker cmd line. + */ + public static Iterable<LinkerInput> mergeInputsCmdLine(NestedSet<LibraryToLink> inputs, + boolean globalNeedWholeArchive, ArchiveType archiveType) { + return new FilterMembersForLinkIterable(inputs, globalNeedWholeArchive, archiveType, false); + } + + /** + * Add in any object files which are implicitly named as inputs by the linker. + */ + public static Iterable<LinkerInput> mergeInputsDependencies(NestedSet<LibraryToLink> inputs, + boolean globalNeedWholeArchive, ArchiveType archiveType) { + return new FilterMembersForLinkIterable(inputs, globalNeedWholeArchive, archiveType, true); + } + + /** + * On the fly implementation to filter the members. + */ + private static final class FilterMembersForLinkIterable implements Iterable<LinkerInput> { + private final boolean globalNeedWholeArchive; + private final ArchiveType archiveType; + private final boolean deps; + + private final Iterable<LibraryToLink> inputs; + + private FilterMembersForLinkIterable(Iterable<LibraryToLink> inputs, + boolean globalNeedWholeArchive, ArchiveType archiveType, boolean deps) { + this.globalNeedWholeArchive = globalNeedWholeArchive; + this.archiveType = archiveType; + this.deps = deps; + this.inputs = CollectionUtils.makeImmutable(inputs); + } + + @Override + public Iterator<LinkerInput> iterator() { + return new FilterMembersForLinkIterator(inputs.iterator(), globalNeedWholeArchive, + archiveType, deps); + } + } + + /** + * On the fly implementation to filter the members. + */ + private static final class FilterMembersForLinkIterator extends AbstractIterator<LinkerInput> { + private final boolean globalNeedWholeArchive; + private final ArchiveType archiveType; + private final boolean deps; + + private final Iterator<LibraryToLink> inputs; + private Iterator<LinkerInput> delayList = ImmutableList.<LinkerInput>of().iterator(); + + private FilterMembersForLinkIterator(Iterator<LibraryToLink> inputs, + boolean globalNeedWholeArchive, ArchiveType archiveType, boolean deps) { + this.globalNeedWholeArchive = globalNeedWholeArchive; + this.archiveType = archiveType; + this.deps = deps; + this.inputs = inputs; + } + + @Override + protected LinkerInput computeNext() { + if (delayList.hasNext()) { + return delayList.next(); + } + + while (inputs.hasNext()) { + LibraryToLink inputLibrary = inputs.next(); + Artifact input = inputLibrary.getArtifact(); + String name = input.getFilename(); + + // True if the linker might use the members of this file, i.e., if the file is a thin or + // start_end_lib archive (aka static library). Also check if the library contains object + // files - otherwise getObjectFiles returns null, which would lead to an NPE in + // simpleLinkerInputs. + boolean needMembersForLink = archiveType != ArchiveType.FAT + && ARCHIVE_LIBRARY_FILETYPES.matches(name) && inputLibrary.containsObjectFiles(); + + // True if we will pass the members instead of the original archive. + boolean passMembersToLinkCmd = needMembersForLink + && (globalNeedWholeArchive || LINK_LIBRARY_FILETYPES.matches(name)); + + // If deps is false (when computing the inputs to be passed on the command line), then it's + // an if-then-else, i.e., the passMembersToLinkCmd flag decides whether to pass the object + // files or the archive itself. This flag in turn is based on whether the archives are fat + // or not (thin archives or start_end_lib) - we never expand fat archives, but we do expand + // non-fat archives if we need whole-archives for the entire link, or for the specific + // library (i.e., if alwayslink=1). + // + // If deps is true (when computing the inputs to be passed to the action as inputs), then it + // becomes more complicated. We always need to pass the members for thin and start_end_lib + // archives (needMembersForLink). And we _also_ need to pass the archive file itself unless + // it's a start_end_lib archive (unless it's an alwayslink library). + + // A note about ordering: the order in which the object files and the library are returned + // does not currently matter - this code results in the library returned first, and the + // object files returned after, but only if both are returned, which can only happen if + // deps is true, in which case this code only computes the list of inputs for the link + // action (so the order isn't critical). + if (passMembersToLinkCmd || (deps && needMembersForLink)) { + delayList = LinkerInputs.simpleLinkerInputs(inputLibrary.getObjectFiles()).iterator(); + } + + if (!(passMembersToLinkCmd || (deps && useStartEndLib(inputLibrary, archiveType)))) { + return inputLibrary; + } + + if (delayList.hasNext()) { + return delayList.next(); + } + } + return endOfData(); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLine.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLine.java new file mode 100644 index 0000000000..1dccafa8cc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLine.java @@ -0,0 +1,1121 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +/** + * Represents the command line of a linker invocation. It supports executables and dynamic + * libraries as well as static libraries. + */ +@Immutable +public final class LinkCommandLine extends CommandLine { + private final BuildConfiguration configuration; + private final CppConfiguration cppConfiguration; + private final ActionOwner owner; + private final Artifact output; + @Nullable private final Artifact interfaceOutput; + @Nullable private final Artifact symbolCountsOutput; + private final ImmutableList<Artifact> buildInfoHeaderArtifacts; + private final Iterable<? extends LinkerInput> linkerInputs; + private final Iterable<? extends LinkerInput> runtimeInputs; + private final LinkTargetType linkTargetType; + private final LinkStaticness linkStaticness; + private final ImmutableList<String> linkopts; + private final ImmutableSet<String> features; + private final ImmutableMap<Artifact, Artifact> linkstamps; + private final ImmutableList<String> linkstampCompileOptions; + @Nullable private final PathFragment runtimeSolibDir; + private final boolean nativeDeps; + private final boolean useTestOnlyFlags; + private final boolean needWholeArchive; + private final boolean supportsParamFiles; + @Nullable private final Artifact interfaceSoBuilder; + + private LinkCommandLine( + BuildConfiguration configuration, + ActionOwner owner, + Artifact output, + @Nullable Artifact interfaceOutput, + @Nullable Artifact symbolCountsOutput, + ImmutableList<Artifact> buildInfoHeaderArtifacts, + Iterable<? extends LinkerInput> linkerInputs, + Iterable<? extends LinkerInput> runtimeInputs, + LinkTargetType linkTargetType, + LinkStaticness linkStaticness, + ImmutableList<String> linkopts, + ImmutableSet<String> features, + ImmutableMap<Artifact, Artifact> linkstamps, + ImmutableList<String> linkstampCompileOptions, + @Nullable PathFragment runtimeSolibDir, + boolean nativeDeps, + boolean useTestOnlyFlags, + boolean needWholeArchive, + boolean supportsParamFiles, + Artifact interfaceSoBuilder) { + Preconditions.checkArgument(linkTargetType != LinkTargetType.INTERFACE_DYNAMIC_LIBRARY, + "you can't link an interface dynamic library directly"); + if (linkTargetType != LinkTargetType.DYNAMIC_LIBRARY) { + Preconditions.checkArgument(interfaceOutput == null, + "interface output may only be non-null for dynamic library links"); + } + if (linkTargetType.isStaticLibraryLink()) { + Preconditions.checkArgument(linkstamps.isEmpty(), + "linkstamps may only be present on dynamic library or executable links"); + Preconditions.checkArgument(linkStaticness == LinkStaticness.FULLY_STATIC, + "static library link must be static"); + Preconditions.checkArgument(buildInfoHeaderArtifacts.isEmpty(), + "build info headers may only be present on dynamic library or executable links"); + Preconditions.checkArgument(symbolCountsOutput == null, + "the symbol counts output must be null for static links"); + Preconditions.checkArgument(runtimeSolibDir == null, + "the runtime solib directory must be null for static links"); + Preconditions.checkArgument(!nativeDeps, + "the native deps flag must be false for static links"); + Preconditions.checkArgument(!needWholeArchive, + "the need whole archive flag must be false for static links"); + } + + this.configuration = Preconditions.checkNotNull(configuration); + this.cppConfiguration = configuration.getFragment(CppConfiguration.class); + this.owner = Preconditions.checkNotNull(owner); + this.output = Preconditions.checkNotNull(output); + this.interfaceOutput = interfaceOutput; + this.symbolCountsOutput = symbolCountsOutput; + this.buildInfoHeaderArtifacts = Preconditions.checkNotNull(buildInfoHeaderArtifacts); + this.linkerInputs = Preconditions.checkNotNull(linkerInputs); + this.runtimeInputs = Preconditions.checkNotNull(runtimeInputs); + this.linkTargetType = Preconditions.checkNotNull(linkTargetType); + this.linkStaticness = Preconditions.checkNotNull(linkStaticness); + // For now, silently ignore linkopts if this is a static library link. + this.linkopts = linkTargetType.isStaticLibraryLink() + ? ImmutableList.<String>of() + : Preconditions.checkNotNull(linkopts); + this.features = Preconditions.checkNotNull(features); + this.linkstamps = Preconditions.checkNotNull(linkstamps); + this.linkstampCompileOptions = linkstampCompileOptions; + this.runtimeSolibDir = runtimeSolibDir; + this.nativeDeps = nativeDeps; + this.useTestOnlyFlags = useTestOnlyFlags; + this.needWholeArchive = needWholeArchive; + this.supportsParamFiles = supportsParamFiles; + // For now, silently ignore interfaceSoBuilder if we don't build an interface dynamic library. + this.interfaceSoBuilder = + ((linkTargetType == LinkTargetType.DYNAMIC_LIBRARY) && (interfaceOutput != null)) + ? Preconditions.checkNotNull(interfaceSoBuilder, + "cannot build interface dynamic library without builder") + : null; + } + + /** + * Returns an interface shared object output artifact produced during linking. This only returns + * non-null if {@link #getLinkTargetType} is {@code DYNAMIC_LIBRARY} and an interface shared + * object was requested. + */ + @Nullable public Artifact getInterfaceOutput() { + return interfaceOutput; + } + + /** + * Returns an artifact containing the number of symbols used per object file passed to the linker. + * This is currently a gold only feature, and is only produced for executables. If another target + * is being linked, or if symbol counts output is disabled, this will be null. + */ + @Nullable public Artifact getSymbolCountsOutput() { + return symbolCountsOutput; + } + + /** + * Returns the (ordered, immutable) list of header files that contain build info. + */ + public ImmutableList<Artifact> getBuildInfoHeaderArtifacts() { + return buildInfoHeaderArtifacts; + } + + /** + * Returns the (ordered, immutable) list of paths to the linker's input files. + */ + public Iterable<? extends LinkerInput> getLinkerInputs() { + return linkerInputs; + } + + /** + * Returns the runtime inputs to the linker. + */ + public Iterable<? extends LinkerInput> getRuntimeInputs() { + return runtimeInputs; + } + + /** + * Returns the current type of link target set. + */ + public LinkTargetType getLinkTargetType() { + return linkTargetType; + } + + /** + * Returns the "staticness" of the link. + */ + public LinkStaticness getLinkStaticness() { + return linkStaticness; + } + + /** + * Returns the additional linker options for this link. + */ + public ImmutableList<String> getLinkopts() { + return linkopts; + } + + /** + * Returns a (possibly empty) mapping of (C++ source file, .o output file) pairs for source files + * that need to be compiled at link time. + * + * <p>This is used to embed various values from the build system into binaries to identify their + * provenance. + */ + public ImmutableMap<Artifact, Artifact> getLinkstamps() { + return linkstamps; + } + + /** + * Returns the location of the C++ runtime solib symlinks. If null, the C++ dynamic runtime + * libraries either do not exist (because they do not come from the depot) or they are in the + * regular solib directory. + */ + @Nullable public PathFragment getRuntimeSolibDir() { + return runtimeSolibDir; + } + + /** + * Returns true for libraries linked as native dependencies for other languages. + */ + public boolean isNativeDeps() { + return nativeDeps; + } + + /** + * Returns true if this link should use test-specific flags (e.g. $EXEC_ORIGIN as the root for + * finding shared libraries or lazy binding); false by default. See bug "Please use + * $EXEC_ORIGIN instead of $ORIGIN when linking cc_tests" for further context. + */ + public boolean useTestOnlyFlags() { + return useTestOnlyFlags; + } + + /** + * Splits the link command-line into a part to be written to a parameter file, and the remaining + * actual command line to be executed (which references the parameter file). Call {@link + * #canBeSplit} first to check if the command-line can be split. + * + * @throws IllegalStateException if the command-line cannot be split + */ + @VisibleForTesting + final Pair<List<String>, List<String>> splitCommandline(PathFragment paramExecPath) { + Preconditions.checkState(canBeSplit()); + List<String> args = getRawLinkArgv(); + if (linkTargetType.isStaticLibraryLink()) { + // Ar link commands can also generate huge command lines. + List<String> paramFileArgs = args.subList(1, args.size()); + List<String> commandlineArgs = new ArrayList<>(); + commandlineArgs.add(args.get(0)); + + commandlineArgs.add("@" + paramExecPath.getPathString()); + return Pair.of(commandlineArgs, paramFileArgs); + } else { + // Gcc link commands tend to generate humongous commandlines for some targets, which may + // not fit on some remote execution machines. To work around this we will employ the help of + // a parameter file and pass any linker options through it. + List<String> paramFileArgs = new ArrayList<>(); + List<String> commandlineArgs = new ArrayList<>(); + extractArgumentsForParamFile(args, commandlineArgs, paramFileArgs); + + commandlineArgs.add("-Wl,@" + paramExecPath.getPathString()); + return Pair.of(commandlineArgs, paramFileArgs); + } + } + + boolean canBeSplit() { + if (!supportsParamFiles) { + return false; + } + switch (linkTargetType) { + // We currently can't split dynamic library links if they have interface outputs. That was + // probably an unintended side effect of the change that introduced interface outputs. + case DYNAMIC_LIBRARY: + return interfaceOutput == null; + case EXECUTABLE: + case STATIC_LIBRARY: + case PIC_STATIC_LIBRARY: + case ALWAYS_LINK_STATIC_LIBRARY: + case ALWAYS_LINK_PIC_STATIC_LIBRARY: + return true; + default: + return false; + } + } + + private static void extractArgumentsForParamFile(List<String> args, List<String> commandlineArgs, + List<String> paramFileArgs) { + // Note, that it is not important that all linker arguments are extracted so that + // they can be moved into a parameter file, but the vast majority should. + commandlineArgs.add(args.get(0)); // gcc command, must not be moved! + int argsSize = args.size(); + for (int i = 1; i < argsSize; i++) { + String arg = args.get(i); + if (arg.equals("-Wl,-no-whole-archive")) { + paramFileArgs.add("-no-whole-archive"); + } else if (arg.equals("-Wl,-whole-archive")) { + paramFileArgs.add("-whole-archive"); + } else if (arg.equals("-Wl,--start-group")) { + paramFileArgs.add("--start-group"); + } else if (arg.equals("-Wl,--end-group")) { + paramFileArgs.add("--end-group"); + } else if (arg.equals("-Wl,--start-lib")) { + paramFileArgs.add("--start-lib"); + } else if (arg.equals("-Wl,--end-lib")) { + paramFileArgs.add("--end-lib"); + } else if (arg.equals("--incremental-unchanged")) { + paramFileArgs.add(arg); + } else if (arg.equals("--incremental-changed")) { + paramFileArgs.add(arg); + } else if (arg.charAt(0) == '-') { + if (arg.startsWith("-l")) { + paramFileArgs.add(arg); + } else { + // Anything else starting with a '-' can stay on the commandline. + commandlineArgs.add(arg); + if (arg.equals("-o")) { + // Special case for '-o': add the following argument as well - it is the output file! + commandlineArgs.add(args.get(++i)); + } + } + } else if (arg.endsWith(".a") || arg.endsWith(".lo") || arg.endsWith(".so") + || arg.endsWith(".ifso") || arg.endsWith(".o") + || CppFileTypes.VERSIONED_SHARED_LIBRARY.matches(arg)) { + // All objects of any kind go into the linker parameters. + paramFileArgs.add(arg); + } else { + // Everything that's left stays conservatively on the commandline. + commandlineArgs.add(arg); + } + } + } + + /** + * Returns a raw link command for the given link invocation, including both command and + * arguments (argv). After any further usage-specific processing, this can be passed to + * {@link #finalizeWithLinkstampCommands} to give the final command line. + * + * @return raw link command line. + */ + public List<String> getRawLinkArgv() { + List<String> argv = new ArrayList<>(); + switch (linkTargetType) { + case EXECUTABLE: + addCppArgv(argv); + break; + + case DYNAMIC_LIBRARY: + if (interfaceOutput != null) { + argv.add(configuration.getShExecutable().getPathString()); + argv.add("-c"); + argv.add("build_iface_so=\"$0\"; impl=\"$1\"; iface=\"$2\"; cmd=\"$3\"; shift 3; " + + "\"$cmd\" \"$@\" && \"$build_iface_so\" \"$impl\" \"$iface\""); + argv.add(interfaceSoBuilder.getExecPathString()); + argv.add(output.getExecPathString()); + argv.add(interfaceOutput.getExecPathString()); + } + addCppArgv(argv); + // -pie is not compatible with -shared and should be + // removed when the latter is part of the link command. Should we need to further + // distinguish between shared libraries and executables, we could add additional + // command line / CROSSTOOL flags that distinguish them. But as long as this is + // the only relevant use case we're just special-casing it here. + Iterables.removeIf(argv, Predicates.equalTo("-pie")); + break; + + case STATIC_LIBRARY: + case PIC_STATIC_LIBRARY: + case ALWAYS_LINK_STATIC_LIBRARY: + case ALWAYS_LINK_PIC_STATIC_LIBRARY: + // The static library link command follows this template: + // ar <cmd> <output_archive> <input_files...> + argv.add(cppConfiguration.getArExecutable().getPathString()); + argv.addAll( + cppConfiguration.getArFlags(cppConfiguration.archiveType() == Link.ArchiveType.THIN)); + argv.add(output.getExecPathString()); + addInputFileLinkOptions(argv, /*needWholeArchive=*/false, + /*includeLinkopts=*/false); + break; + + default: + throw new IllegalArgumentException(); + } + + // Fission mode: debug info is in .dwo files instead of .o files. Inform the linker of this. + if (!linkTargetType.isStaticLibraryLink() && cppConfiguration.useFission()) { + argv.add("-Wl,--gdb-index"); + } + + return argv; + } + + @Override + public List<String> arguments() { + return finalizeWithLinkstampCommands(getRawLinkArgv()); + } + + /** + * Takes a raw link command line and gives the final link command that will + * also first compile any linkstamps necessary. Elements of rawLinkArgv are + * shell-escaped. + * + * @param rawLinkArgv raw link command line + * + * @return final link command line suitable for execution + */ + public List<String> finalizeWithLinkstampCommands(List<String> rawLinkArgv) { + return addLinkstampingToCommand(getLinkstampCompileCommands(""), rawLinkArgv, true); + } + + /** + * Takes a raw link command line and gives the final link command that will also first compile any + * linkstamps necessary. Elements of rawLinkArgv are not shell-escaped. + * + * @param rawLinkArgv raw link command line + * @param outputPrefix prefix to add before the linkstamp outputs' exec paths + * + * @return final link command line suitable for execution + */ + public List<String> finalizeAlreadyEscapedWithLinkstampCommands( + List<String> rawLinkArgv, String outputPrefix) { + return addLinkstampingToCommand(getLinkstampCompileCommands(outputPrefix), rawLinkArgv, false); + } + + /** + * Adds linkstamp compilation to the (otherwise) fully specified link + * command if {@link #getLinkstamps} is non-empty. + * + * <p>Linkstamps were historically compiled implicitly as part of the link + * command, but implicit compilation doesn't guarantee consistent outputs. + * For example, the command "gcc input.o input.o foo/linkstamp.cc -o myapp" + * causes gcc to implicitly run "gcc foo/linkstamp.cc -o /tmp/ccEtJHDB.o", + * for some internally decided output path /tmp/ccEtJHDB.o, then add that path + * to the linker's command line options. The name of this path can change + * even between equivalently specified gcc invocations. + * + * <p>So now we explicitly compile these files in their own command + * invocations before running the link command, thus giving us direct + * control over the naming of their outputs. This method adds those extra + * steps as necessary. + * @param linkstampCommands individual linkstamp compilation commands + * @param linkCommand the complete list of link command arguments (after + * .params file compacting) for an invocation + * @param escapeArgs if true, linkCommand arguments are shell escaped. if + * false, arguments are returned as-is + * + * @return The original argument list if no linkstamps compilation commands + * are given, otherwise an expanded list that adds the linkstamp + * compilation commands and funnels their outputs into the link step. + * Note that these outputs only need to persist for the duration of + * the link step. + */ + private static List<String> addLinkstampingToCommand( + List<String> linkstampCommands, + List<String> linkCommand, + boolean escapeArgs) { + if (linkstampCommands.isEmpty()) { + return linkCommand; + } else { + List<String> batchCommand = Lists.newArrayListWithCapacity(3); + batchCommand.add("/bin/bash"); + batchCommand.add("-c"); + batchCommand.add( + Joiner.on(" && ").join(linkstampCommands) + " && " + + (escapeArgs + ? ShellEscaper.escapeJoinAll(linkCommand) + : Joiner.on(" ").join(linkCommand))); + return ImmutableList.copyOf(batchCommand); + } + } + + /** + * Computes, for each C++ source file in + * {@link #getLinkstamps}, the command necessary to compile + * that file such that the output is correctly fed into the link command. + * + * <p>As these options (as well as all others) are taken into account when + * computing the action key, they do not directly contain volatile build + * information to avoid unnecessary relinking. Instead this information is + * passed as an additional header generated by + * {@link com.google.devtools.build.lib.rules.cpp.WriteBuildInfoHeaderAction}. + * + * @param outputPrefix prefix to add before the linkstamp outputs' exec paths + * @return a list of shell-escaped compiler commmands, one for each entry + * in {@link #getLinkstamps} + */ + public List<String> getLinkstampCompileCommands(String outputPrefix) { + if (linkstamps.isEmpty()) { + return ImmutableList.of(); + } + + String compilerCommand = cppConfiguration.getCppExecutable().getPathString(); + List<String> commands = Lists.newArrayListWithCapacity(linkstamps.size()); + + for (Map.Entry<Artifact, Artifact> linkstamp : linkstamps.entrySet()) { + List<String> optionList = new ArrayList<>(); + + // Defines related to the build info are read from generated headers. + for (Artifact header : buildInfoHeaderArtifacts) { + optionList.add("-include"); + optionList.add(header.getExecPathString()); + } + + String labelReplacement = Matcher.quoteReplacement( + isSharedNativeLibrary() ? output.getExecPathString() : Label.print(owner.getLabel())); + String outputPathReplacement = Matcher.quoteReplacement( + output.getExecPathString()); + for (String option : linkstampCompileOptions) { + optionList.add(option + .replaceAll(Pattern.quote("${LABEL}"), labelReplacement) + .replaceAll(Pattern.quote("${OUTPUT_PATH}"), outputPathReplacement)); + } + + optionList.add("-DGPLATFORM=\"" + cppConfiguration + "\""); + + // Needed to find headers included from linkstamps. + optionList.add("-I."); + + // Add sysroot. + PathFragment sysroot = cppConfiguration.getSysroot(); + if (sysroot != null) { + optionList.add("--sysroot=" + sysroot.getPathString()); + } + + // Add toolchain compiler options. + optionList.addAll(cppConfiguration.getCompilerOptions(features)); + optionList.addAll(cppConfiguration.getCOptions()); + optionList.addAll(cppConfiguration.getUnfilteredCompilerOptions(features)); + + // For dynamic libraries, produce position independent code. + if (linkTargetType == LinkTargetType.DYNAMIC_LIBRARY + && cppConfiguration.toolchainNeedsPic()) { + optionList.add("-fPIC"); + } + + // Stamp FDO builds with FDO subtype string + String fdoBuildStamp = CppHelper.getFdoBuildStamp(cppConfiguration); + if (fdoBuildStamp != null) { + optionList.add("-D" + CppConfiguration.FDO_STAMP_MACRO + "=\"" + fdoBuildStamp + "\""); + } + + // Add the compilation target. + optionList.add("-c"); + optionList.add(linkstamp.getKey().getExecPathString()); + + // Assemble the final command, exempting outputPrefix from shell escaping. + commands.add(compilerCommand + " " + + ShellEscaper.escapeJoinAll(optionList) + + " -o " + + outputPrefix + + ShellEscaper.escapeString(linkstamp.getValue().getExecPathString())); + } + + return commands; + } + + /** + * Determine the arguments to pass to the C++ compiler when linking. + * Add them to the {@code argv} parameter. + */ + private void addCppArgv(List<String> argv) { + argv.add(cppConfiguration.getCppExecutable().getPathString()); + + // When using gold to link an executable, output the number of used and unused symbols. + if (symbolCountsOutput != null) { + argv.add("-Wl,--print-symbol-counts=" + symbolCountsOutput.getExecPathString()); + } + + if (linkTargetType == LinkTargetType.DYNAMIC_LIBRARY) { + argv.add("-shared"); + } + + // Add the outputs of any associated linkstamp compilations. + for (Artifact linkstampOutput : linkstamps.values()) { + argv.add(linkstampOutput.getExecPathString()); + } + + boolean fullyStatic = (linkStaticness == LinkStaticness.FULLY_STATIC); + boolean mostlyStatic = (linkStaticness == LinkStaticness.MOSTLY_STATIC); + boolean sharedLinkopts = + linkTargetType == LinkTargetType.DYNAMIC_LIBRARY + || linkopts.contains("-shared") + || cppConfiguration.getLinkOptions().contains("-shared"); + + if (output != null) { + argv.add("-o"); + String execpath = output.getExecPathString(); + if (mostlyStatic + && linkTargetType == LinkTargetType.EXECUTABLE + && cppConfiguration.skipStaticOutputs()) { + // Linked binary goes to /dev/null; bogus dependency info in its place. + Collections.addAll(argv, "/dev/null", "-MMD", "-MF", execpath); // thanks Ambrose + } else { + argv.add(execpath); + } + } + + addInputFileLinkOptions(argv, needWholeArchive, /*includeLinkopts=*/true); + + // Extra toolchain link options based on the output's link staticness. + if (fullyStatic) { + argv.addAll(cppConfiguration.getFullyStaticLinkOptions(features, sharedLinkopts)); + } else if (mostlyStatic) { + argv.addAll(cppConfiguration.getMostlyStaticLinkOptions(features, sharedLinkopts)); + } else { + argv.addAll(cppConfiguration.getDynamicLinkOptions(features, sharedLinkopts)); + } + + // Extra test-specific link options. + if (useTestOnlyFlags) { + argv.addAll(cppConfiguration.getTestOnlyLinkOptions()); + } + + if (configuration.isCodeCoverageEnabled()) { + argv.add("-lgcov"); + } + + if (linkTargetType == LinkTargetType.EXECUTABLE && cppConfiguration.forcePic()) { + argv.add("-pie"); + } + + argv.addAll(cppConfiguration.getLinkOptions()); + argv.addAll(cppConfiguration.getFdoSupport().getLinkOptions()); + } + + private static boolean isDynamicLibrary(LinkerInput linkInput) { + Artifact libraryArtifact = linkInput.getArtifact(); + String name = libraryArtifact.getFilename(); + return Link.SHARED_LIBRARY_FILETYPES.matches(name) && name.startsWith("lib"); + } + + private boolean isSharedNativeLibrary() { + return nativeDeps && cppConfiguration.shareNativeDeps(); + } + + /** + * When linking a shared library fully or mostly static then we need to link in + * *all* dependent files, not just what the shared library needs for its own + * code. This is done by wrapping all objects/libraries with + * -Wl,-whole-archive and -Wl,-no-whole-archive. For this case the + * globalNeedWholeArchive parameter must be set to true. Otherwise only + * library objects (.lo) need to be wrapped with -Wl,-whole-archive and + * -Wl,-no-whole-archive. + */ + private void addInputFileLinkOptions(List<String> argv, boolean globalNeedWholeArchive, + boolean includeLinkopts) { + // The Apple ld doesn't support -whole-archive/-no-whole-archive. It + // does have -all_load/-noall_load, but -all_load is a global setting + // that affects all subsequent files, and -noall_load is simply ignored. + // TODO(bazel-team): Not sure what the implications of this are, other than + // bloated binaries. + boolean macosx = cppConfiguration.getTargetLibc().equals("macosx"); + if (globalNeedWholeArchive) { + argv.add(macosx ? "-Wl,-all_load" : "-Wl,-whole-archive"); + } + + // Used to collect -L and -Wl,-rpath options, ensuring that each used only once. + Set<String> libOpts = new LinkedHashSet<>(); + + // List of command line parameters to link input files (either directly or using -l). + List<String> linkerInputs = new ArrayList<>(); + + // List of command line parameters that need to be placed *outside* of + // --whole-archive ... --no-whole-archive. + List<String> noWholeArchiveInputs = new ArrayList<>(); + + PathFragment solibDir = configuration.getBinDirectory().getExecPath() + .getRelative(cppConfiguration.getSolibDirectory()); + String runtimeSolibName = runtimeSolibDir != null ? runtimeSolibDir.getBaseName() : null; + boolean runtimeRpath = runtimeSolibDir != null + && (linkTargetType == LinkTargetType.DYNAMIC_LIBRARY + || (linkTargetType == LinkTargetType.EXECUTABLE + && linkStaticness == LinkStaticness.DYNAMIC)); + + String rpathRoot = null; + List<String> runtimeRpathEntries = new ArrayList<>(); + + if (output != null) { + String origin = + useTestOnlyFlags && cppConfiguration.supportsExecOrigin() ? "$EXEC_ORIGIN/" : "$ORIGIN/"; + if (runtimeRpath) { + runtimeRpathEntries.add("-Wl,-rpath," + origin + runtimeSolibName + "/"); + } + + // Calculate the correct relative value for the "-rpath" link option (which sets + // the search path for finding shared libraries). + if (isSharedNativeLibrary()) { + // For shared native libraries, special symlinking is applied to ensure C++ + // runtimes are available under $ORIGIN/_solib_[arch]. So we set the RPATH to find + // them. + // + // Note that we have to do this because $ORIGIN points to different paths for + // different targets. In other words, blaze-bin/d1/d2/d3/a_shareddeps.so and + // blaze-bin/d4/b_shareddeps.so have different path depths. The first could + // reference a standard blaze-bin/_solib_[arch] via $ORIGIN/../../../_solib[arch], + // and the second could use $ORIGIN/../_solib_[arch]. But since this is a shared + // artifact, both are symlinks to the same place, so + // there's no *one* RPATH setting that fits all targets involved in the sharing. + rpathRoot = "-Wl,-rpath," + origin + ":" + + origin + cppConfiguration.getSolibDirectory() + "/"; + if (runtimeRpath) { + runtimeRpathEntries.add("-Wl,-rpath," + origin + "../" + runtimeSolibName + "/"); + } + } else { + // For all other links, calculate the relative path from the output file to _solib_[arch] + // (the directory where all shared libraries are stored, which resides under the blaze-bin + // directory. In other words, given blaze-bin/my/package/binary, rpathRoot would be + // "../../_solib_[arch]". + if (runtimeRpath) { + runtimeRpathEntries.add("-Wl,-rpath," + origin + + Strings.repeat("../", output.getRootRelativePath().segmentCount() - 1) + + runtimeSolibName + "/"); + } + + rpathRoot = "-Wl,-rpath," + + origin + Strings.repeat("../", output.getRootRelativePath().segmentCount() - 1) + + cppConfiguration.getSolibDirectory() + "/"; + + if (nativeDeps) { + // We also retain the $ORIGIN/ path to solibs that are in _solib_<arch>, as opposed to + // the package directory) + if (runtimeRpath) { + runtimeRpathEntries.add("-Wl,-rpath," + origin + "../" + runtimeSolibName + "/"); + } + rpathRoot += ":" + origin; + } + } + } + + boolean includeSolibDir = false; + + for (LinkerInput input : getLinkerInputs()) { + if (isDynamicLibrary(input)) { + PathFragment libDir = input.getArtifact().getExecPath().getParentDirectory(); + Preconditions.checkState( + libDir.startsWith(solibDir), + "Artifact '%s' is not under directory '%s'.", input.getArtifact(), solibDir); + if (libDir.equals(solibDir)) { + includeSolibDir = true; + } + addDynamicInputLinkOptions(input, linkerInputs, libOpts, solibDir, rpathRoot); + } else { + addStaticInputLinkOptions(input, linkerInputs); + } + } + + boolean includeRuntimeSolibDir = false; + + for (LinkerInput input : runtimeInputs) { + List<String> optionsList = globalNeedWholeArchive + ? noWholeArchiveInputs + : linkerInputs; + + if (isDynamicLibrary(input)) { + PathFragment libDir = input.getArtifact().getExecPath().getParentDirectory(); + Preconditions.checkState(runtimeSolibDir != null && libDir.equals(runtimeSolibDir), + "Artifact '%s' is not under directory '%s'.", input.getArtifact(), solibDir); + includeRuntimeSolibDir = true; + addDynamicInputLinkOptions(input, optionsList, libOpts, solibDir, rpathRoot); + } else { + addStaticInputLinkOptions(input, optionsList); + } + } + + // rpath ordering matters for performance; first add the one where most libraries are found. + if (includeSolibDir && rpathRoot != null) { + argv.add(rpathRoot); + } + if (includeRuntimeSolibDir) { + argv.addAll(runtimeRpathEntries); + } + argv.addAll(libOpts); + + // Need to wrap static libraries with whole-archive option + for (String option : linkerInputs) { + if (!globalNeedWholeArchive && Link.LINK_LIBRARY_FILETYPES.matches(option)) { + argv.add(macosx ? "-Wl,-all_load" : "-Wl,-whole-archive"); + argv.add(option); + argv.add(macosx ? "-Wl,-noall_load" : "-Wl,-no-whole-archive"); + } else { + argv.add(option); + } + } + + if (globalNeedWholeArchive) { + argv.add(macosx ? "-Wl,-noall_load" : "-Wl,-no-whole-archive"); + argv.addAll(noWholeArchiveInputs); + } + + if (includeLinkopts) { + /* + * For backwards compatibility, linkopts come _after_ inputFiles. + * This is needed to allow linkopts to contain libraries and + * positional library-related options such as + * -Wl,--begin-group -lfoo -lbar -Wl,--end-group + * or + * -Wl,--as-needed -lfoo -Wl,--no-as-needed + * + * As for the relative order of the three different flavours of linkopts + * (global defaults, per-target linkopts, and command-line linkopts), + * we have no idea what the right order should be, or if anyone cares. + */ + argv.addAll(linkopts); + } + } + + /** + * Adds command-line options for a dynamic library input file into + * options and libOpts. + */ + private void addDynamicInputLinkOptions(LinkerInput input, List<String> options, + Set<String> libOpts, PathFragment solibDir, String rpathRoot) { + Preconditions.checkState(isDynamicLibrary(input)); + Preconditions.checkState( + !Link.useStartEndLib(input, cppConfiguration.archiveType())); + + Artifact inputArtifact = input.getArtifact(); + PathFragment libDir = inputArtifact.getExecPath().getParentDirectory(); + if (rpathRoot != null + && !libDir.equals(solibDir) + && (runtimeSolibDir == null || !runtimeSolibDir.equals(libDir))) { + String dotdots = ""; + PathFragment commonParent = solibDir; + while (!libDir.startsWith(commonParent)) { + dotdots += "../"; + commonParent = commonParent.getParentDirectory(); + } + + libOpts.add(rpathRoot + dotdots + libDir.relativeTo(commonParent).getPathString()); + } + + libOpts.add("-L" + inputArtifact.getExecPath().getParentDirectory().getPathString()); + + String name = inputArtifact.getFilename(); + if (CppFileTypes.SHARED_LIBRARY.matches(name)) { + String libName = name.replaceAll("(^lib|\\.so$)", ""); + options.add("-l" + libName); + } else { + // Interface shared objects have a non-standard extension + // that the linker won't be able to find. So use the + // filename directly rather than a -l option. Since the + // library has an SONAME attribute, this will work fine. + options.add(inputArtifact.getExecPathString()); + } + } + + /** + * Adds command-line options for a static library or non-library input + * into options. + */ + private void addStaticInputLinkOptions(LinkerInput input, List<String> options) { + Preconditions.checkState(!isDynamicLibrary(input)); + + // start-lib/end-lib library: adds its input object files. + if (Link.useStartEndLib(input, cppConfiguration.archiveType())) { + Iterable<Artifact> archiveMembers = input.getObjectFiles(); + if (!Iterables.isEmpty(archiveMembers)) { + options.add("-Wl,--start-lib"); + for (Artifact member : archiveMembers) { + options.add(member.getExecPathString()); + } + options.add("-Wl,--end-lib"); + } + // For anything else, add the input directly. + } else { + Artifact inputArtifact = input.getArtifact(); + if (input.isFake()) { + options.add(Link.FAKE_OBJECT_PREFIX + inputArtifact.getExecPathString()); + } else { + options.add(inputArtifact.getExecPathString()); + } + } + } + + /** + * A builder for a {@link LinkCommandLine}. + */ + public static final class Builder { + // TODO(bazel-team): Pass this in instead of having it here. Maybe move to cc_toolchain. + private static final ImmutableList<String> DEFAULT_LINKSTAMP_OPTIONS = ImmutableList.of( + // G3_VERSION_INFO and G3_TARGET_NAME are C string literals that normally + // contain the label of the target being linked. However, they are set + // differently when using shared native deps. In that case, a single .so file + // is shared by multiple targets, and its contents cannot depend on which + // target(s) were specified on the command line. So in that case we have + // to use the (obscure) name of the .so file instead, or more precisely + // the path of the .so file relative to the workspace root. + "-DG3_VERSION_INFO=\"${LABEL}\"", + "-DG3_TARGET_NAME=\"${LABEL}\"", + + // G3_BUILD_TARGET is a C string literal containing the output of this + // link. (An undocumented and untested invariant is that G3_BUILD_TARGET is the location of + // the executable, either absolutely, or relative to the directory part of BUILD_INFO.) + "-DG3_BUILD_TARGET=\"${OUTPUT_PATH}\""); + + private final BuildConfiguration configuration; + private final ActionOwner owner; + + @Nullable private Artifact output; + @Nullable private Artifact interfaceOutput; + @Nullable private Artifact symbolCountsOutput; + private ImmutableList<Artifact> buildInfoHeaderArtifacts = ImmutableList.of(); + private Iterable<? extends LinkerInput> linkerInputs = ImmutableList.of(); + private Iterable<? extends LinkerInput> runtimeInputs = ImmutableList.of(); + @Nullable private LinkTargetType linkTargetType; + private LinkStaticness linkStaticness = LinkStaticness.FULLY_STATIC; + private ImmutableList<String> linkopts = ImmutableList.of(); + private ImmutableSet<String> features = ImmutableSet.of(); + private ImmutableMap<Artifact, Artifact> linkstamps = ImmutableMap.of(); + private List<String> linkstampCompileOptions = new ArrayList<>(); + @Nullable private PathFragment runtimeSolibDir; + private boolean nativeDeps; + private boolean useTestOnlyFlags; + private boolean needWholeArchive; + private boolean supportsParamFiles; + @Nullable private Artifact interfaceSoBuilder; + + public Builder(BuildConfiguration configuration, ActionOwner owner) { + this.configuration = configuration; + this.owner = owner; + } + + public Builder(RuleContext ruleContext) { + this(ruleContext.getConfiguration(), ruleContext.getActionOwner()); + } + + public LinkCommandLine build() { + ImmutableList<String> actualLinkstampCompileOptions; + if (linkstampCompileOptions.isEmpty()) { + actualLinkstampCompileOptions = DEFAULT_LINKSTAMP_OPTIONS; + } else { + actualLinkstampCompileOptions = ImmutableList.copyOf( + Iterables.concat(DEFAULT_LINKSTAMP_OPTIONS, linkstampCompileOptions)); + } + return new LinkCommandLine(configuration, owner, output, interfaceOutput, + symbolCountsOutput, buildInfoHeaderArtifacts, linkerInputs, runtimeInputs, linkTargetType, + linkStaticness, linkopts, features, linkstamps, actualLinkstampCompileOptions, + runtimeSolibDir, nativeDeps, useTestOnlyFlags, needWholeArchive, supportsParamFiles, + interfaceSoBuilder); + } + + /** + * Sets the type of the link. It is an error to try to set this to {@link + * LinkTargetType#INTERFACE_DYNAMIC_LIBRARY}. Note that all the static target types (see {@link + * LinkTargetType#isStaticLibraryLink}) are equivalent, and there is no check that the output + * artifact matches the target type extension. + */ + public Builder setLinkTargetType(LinkTargetType linkTargetType) { + Preconditions.checkArgument(linkTargetType != LinkTargetType.INTERFACE_DYNAMIC_LIBRARY); + this.linkTargetType = linkTargetType; + return this; + } + + /** + * Sets the primary output artifact. This must be called before calling {@link #build}. + */ + public Builder setOutput(Artifact output) { + this.output = output; + return this; + } + + /** + * Sets a list of linker inputs. These get turned into linker options depending on the + * staticness and the target type. This call makes an immutable copy of the inputs, if the + * provided Iterable isn't already immutable (see {@link CollectionUtils#makeImmutable}). + */ + public Builder setLinkerInputs(Iterable<LinkerInput> linkerInputs) { + this.linkerInputs = CollectionUtils.makeImmutable(linkerInputs); + return this; + } + + public Builder setRuntimeInputs(ImmutableList<LinkerInput> runtimeInputs) { + this.runtimeInputs = runtimeInputs; + return this; + } + + /** + * Sets the additional interface output artifact, which is only used for dynamic libraries. The + * {@link #build} method throws an exception if the target type is not {@link + * LinkTargetType#DYNAMIC_LIBRARY}. + */ + public Builder setInterfaceOutput(Artifact interfaceOutput) { + this.interfaceOutput = interfaceOutput; + return this; + } + + /** + * Sets an additional output artifact that contains symbol counts. The {@link #build} method + * throws an exception if this is non-null for a static link (see + * {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setSymbolCountsOutput(Artifact symbolCountsOutput) { + this.symbolCountsOutput = symbolCountsOutput; + return this; + } + + /** + * Sets the linker options. These are passed to the linker in addition to the other linker + * options like linker inputs, symbol count options, etc. The {@link #build} method + * throws an exception if the linker options are non-empty for a static link (see {@link + * LinkTargetType#isStaticLibraryLink}). + */ + public Builder setLinkopts(ImmutableList<String> linkopts) { + this.linkopts = linkopts; + return this; + } + + /** + * Sets how static the link is supposed to be. For static target types (see {@link + * LinkTargetType#isStaticLibraryLink}), the {@link #build} method throws an exception if this + * is not {@link LinkStaticness#FULLY_STATIC}. The default setting is {@link + * LinkStaticness#FULLY_STATIC}. + */ + public Builder setLinkStaticness(LinkStaticness linkStaticness) { + this.linkStaticness = linkStaticness; + return this; + } + + /** + * Sets the binary that should be used to create the interface output for a dynamic library. + * This is ignored unless the target type is {@link LinkTargetType#DYNAMIC_LIBRARY} and an + * interface output artifact is specified. + */ + public Builder setInterfaceSoBuilder(Artifact interfaceSoBuilder) { + this.interfaceSoBuilder = interfaceSoBuilder; + return this; + } + + /** + * Sets the linkstamps. Linkstamps are additional C++ source files that are compiled as part of + * the link command. The {@link #build} method throws an exception if the linkstamps are + * non-empty for a static link (see {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setLinkstamps(ImmutableMap<Artifact, Artifact> linkstamps) { + this.linkstamps = linkstamps; + return this; + } + + /** + * Adds the given C++ compiler options to the list of options passed to the linkstamp + * compilation. + */ + public Builder addLinkstampCompileOptions(List<String> linkstampCompileOptions) { + this.linkstampCompileOptions.addAll(linkstampCompileOptions); + return this; + } + + /** + * The build info header artifacts are generated header files that are used for link stamping. + * The {@link #build} method throws an exception if the build info header artifacts are + * non-empty for a static link (see {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setBuildInfoHeaderArtifacts(ImmutableList<Artifact> buildInfoHeaderArtifacts) { + this.buildInfoHeaderArtifacts = buildInfoHeaderArtifacts; + return this; + } + + /** + * Sets the features enabled for the rule. + */ + public Builder setFeatures(ImmutableSet<String> features) { + this.features = features; + return this; + } + + /** + * Sets the directory of the dynamic runtime libraries, which is added to the rpath. The {@link + * #build} method throws an exception if the runtime dir is non-null for a static link (see + * {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setRuntimeSolibDir(PathFragment runtimeSolibDir) { + this.runtimeSolibDir = runtimeSolibDir; + return this; + } + + /** + * Whether the resulting library is intended to be used as a native library from another + * programming language. This influences the rpath. The {@link #build} method throws an + * exception if this is true for a static link (see {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setNativeDeps(boolean nativeDeps) { + this.nativeDeps = nativeDeps; + return this; + } + + /** + * Sets whether to use test-specific linker flags, e.g. {@code $EXEC_ORIGIN} instead of + * {@code $ORIGIN} in the rpath or lazy binding. + */ + public Builder setUseTestOnlyFlags(boolean useTestOnlyFlags) { + this.useTestOnlyFlags = useTestOnlyFlags; + return this; + } + + public Builder setNeedWholeArchive(boolean needWholeArchive) { + this.needWholeArchive = needWholeArchive; + return this; + } + + public Builder setSupportsParamFiles(boolean supportsParamFiles) { + this.supportsParamFiles = supportsParamFiles; + return this; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkStrategy.java new file mode 100644 index 0000000000..4f7673ecb2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkStrategy.java @@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +/** + * A strategy for executing {@link CppLinkAction}s. + * + * <p>The linker commands, e.g. "ar", are not necessary functional, i.e. + * they may mutate the output file rather than overwriting it. + * To avoid this, we need to delete the output file before invoking the + * command. That must be done by the classes that extend this class. + */ +public abstract class LinkStrategy implements CppLinkActionContext { + public LinkStrategy() { + } + + /** The strategy name, preferably suitable for passing to --link_strategy. */ + public abstract String linkStrategyName(); + + @Override + public String strategyLocality(CppLinkAction execOwner) { + return linkStrategyName(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInput.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInput.java new file mode 100644 index 0000000000..15a8b90c7f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInput.java @@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; + +/** + * Something that appears on the command line of the linker. Since we sometimes expand archive + * files to their constituent object files, we need to keep information whether a certain file + * contains embedded objects and if so, the list of the object files themselves. + */ +public interface LinkerInput { + /** + * Returns the artifact that is the input of the linker. + */ + Artifact getArtifact(); + + /** + * Returns the original library to link. If this library is a solib symlink, returns the + * artifact the symlink points to, otherwise, the library itself. + */ + Artifact getOriginalLibraryArtifact(); + + /** + * Whether the input artifact contains object files or is opaque. + */ + boolean containsObjectFiles(); + + /** + * Returns whether the input artifact is a fake object file or not. + */ + boolean isFake(); + + /** + * Return the list of object files included in the input artifact, if there are any. It is + * legal to call this only when {@link #containsObjectFiles()} returns true. + */ + Iterable<Artifact> getObjectFiles(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInputs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInputs.java new file mode 100644 index 0000000000..24120ce44f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInputs.java @@ -0,0 +1,353 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.concurrent.ThreadSafety; + +/** + * Factory for creating new {@link LinkerInput} objects. + */ +public abstract class LinkerInputs { + /** + * An opaque linker input that is not a library, for example a linker script or an individual + * object file. + */ + @ThreadSafety.Immutable + public static class SimpleLinkerInput implements LinkerInput { + private final Artifact artifact; + + public SimpleLinkerInput(Artifact artifact) { + this.artifact = Preconditions.checkNotNull(artifact); + } + + @Override + public Artifact getArtifact() { + return artifact; + } + + @Override + public Artifact getOriginalLibraryArtifact() { + return artifact; + } + + @Override + public boolean containsObjectFiles() { + return false; + } + + @Override + public boolean isFake() { + return false; + } + + @Override + public Iterable<Artifact> getObjectFiles() { + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof SimpleLinkerInput)) { + return false; + } + + SimpleLinkerInput other = (SimpleLinkerInput) that; + return artifact.equals(other.artifact) && isFake() == other.isFake(); + } + + @Override + public int hashCode() { + return artifact.hashCode(); + } + + @Override + public String toString() { + return "SimpleLinkerInput(" + artifact.toString() + ")"; + } + } + + /** + * A linker input that is a fake object file generated by cc_fake_binary. The contained + * artifact must be an object file. + */ + @ThreadSafety.Immutable + private static class FakeLinkerInput extends SimpleLinkerInput { + private FakeLinkerInput(Artifact artifact) { + super(artifact); + Preconditions.checkState(Link.OBJECT_FILETYPES.matches(artifact.getFilename())); + } + + @Override + public boolean isFake() { + return true; + } + } + + /** + * A library the user can link to. This is different from a simple linker input in that it also + * has a library identifier. + */ + public interface LibraryToLink extends LinkerInput { + /** + * Returns whether the library is a solib symlink. + */ + boolean isSolibSymlink(); + } + + /** + * This class represents a solib library symlink. Its library identifier is inherited from + * the library that it links to. + */ + @ThreadSafety.Immutable + public static class SolibLibraryToLink implements LibraryToLink { + private final Artifact solibSymlinkArtifact; + private final Artifact libraryArtifact; + + private SolibLibraryToLink(Artifact solibSymlinkArtifact, Artifact libraryArtifact) { + this.solibSymlinkArtifact = Preconditions.checkNotNull(solibSymlinkArtifact); + this.libraryArtifact = libraryArtifact; + } + + @Override + public String toString() { + return String.format("SolibLibraryToLink(%s -> %s", + solibSymlinkArtifact.toString(), libraryArtifact.toString()); + } + + @Override + public Artifact getArtifact() { + return solibSymlinkArtifact; + } + + @Override + public boolean containsObjectFiles() { + return false; + } + + @Override + public boolean isFake() { + return false; + } + + @Override + public Iterable<Artifact> getObjectFiles() { + throw new IllegalStateException(); + } + + @Override + public Artifact getOriginalLibraryArtifact() { + return libraryArtifact; + } + + @Override + public boolean isSolibSymlink() { + return true; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof SolibLibraryToLink)) { + return false; + } + + SolibLibraryToLink thatSolib = (SolibLibraryToLink) that; + return + solibSymlinkArtifact.equals(thatSolib.solibSymlinkArtifact) && + libraryArtifact.equals(thatSolib.libraryArtifact); + } + + @Override + public int hashCode() { + return solibSymlinkArtifact.hashCode(); + } + } + + /** + * This class represents a library that may contain object files. + */ + @ThreadSafety.Immutable + private static class CompoundLibraryToLink implements LibraryToLink { + private final Artifact libraryArtifact; + private final Iterable<Artifact> objectFiles; + + private CompoundLibraryToLink(Artifact libraryArtifact, Iterable<Artifact> objectFiles) { + this.libraryArtifact = Preconditions.checkNotNull(libraryArtifact); + this.objectFiles = objectFiles == null ? null : CollectionUtils.makeImmutable(objectFiles); + } + + @Override + public String toString() { + return String.format("CompoundLibraryToLink(%s)", libraryArtifact.toString()); + } + + @Override + public Artifact getArtifact() { + return libraryArtifact; + } + + @Override + public Artifact getOriginalLibraryArtifact() { + return libraryArtifact; + } + + @Override + public boolean containsObjectFiles() { + return objectFiles != null; + } + + @Override + public boolean isFake() { + return false; + } + + @Override + public Iterable<Artifact> getObjectFiles() { + Preconditions.checkNotNull(objectFiles); + return objectFiles; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof CompoundLibraryToLink)) { + return false; + } + + return libraryArtifact.equals(((CompoundLibraryToLink) that).libraryArtifact); + } + + @Override + public int hashCode() { + return libraryArtifact.hashCode(); + } + + @Override + public boolean isSolibSymlink() { + return false; + } + } + + ////////////////////////////////////////////////////////////////////////////////////// + // Public factory constructors: + ////////////////////////////////////////////////////////////////////////////////////// + + /** + * Creates linker input objects for non-library files. + */ + public static Iterable<LinkerInput> simpleLinkerInputs(Iterable<Artifact> input) { + return Iterables.transform(input, new Function<Artifact, LinkerInput>() { + @Override + public LinkerInput apply(Artifact artifact) { + return simpleLinkerInput(artifact); + } + }); + } + + /** + * Creates a linker input for which we do not know what objects files it consists of. + */ + public static LinkerInput simpleLinkerInput(Artifact artifact) { + // This precondition check was in place and *most* of the tests passed with them; the only + // exception is when you mention a generated .a file in the srcs of a cc_* rule. + // Preconditions.checkArgument(!ARCHIVE_LIBRARY_FILETYPES.contains(artifact.getFileType())); + return new SimpleLinkerInput(artifact); + } + + /** + * Creates a fake linker input. The artifact must be an object file. + */ + public static LinkerInput fakeLinkerInput(Artifact artifact) { + return new FakeLinkerInput(artifact); + } + + /** + * Creates input libraries for which we do not know what objects files it consists of. + */ + public static Iterable<LibraryToLink> opaqueLibrariesToLink(Iterable<Artifact> input) { + return Iterables.transform(input, new Function<Artifact, LibraryToLink>() { + @Override + public LibraryToLink apply(Artifact artifact) { + return opaqueLibraryToLink(artifact); + } + }); + } + + /** + * Creates a solib library symlink from the given artifact. + */ + public static LibraryToLink solibLibraryToLink(Artifact solibSymlink, Artifact original) { + return new SolibLibraryToLink(solibSymlink, original); + } + + /** + * Creates an input library for which we do not know what objects files it consists of. + */ + public static LibraryToLink opaqueLibraryToLink(Artifact artifact) { + // This precondition check was in place and *most* of the tests passed with them; the only + // exception is when you mention a generated .a file in the srcs of a cc_* rule. + // It was very useful for proving that this actually works, though. + // Preconditions.checkArgument( + // !(artifact.getGeneratingAction() instanceof CppLinkAction) || + // !Link.ARCHIVE_LIBRARY_FILETYPES.contains(artifact.getFileType())); + return new CompoundLibraryToLink(artifact, null); + } + + /** + * Creates a library to link with the specified object files. + */ + public static LibraryToLink newInputLibrary(Artifact library, Iterable<Artifact> objectFiles) { + return new CompoundLibraryToLink(library, objectFiles); + } + + private static final Function<LibraryToLink, Artifact> LIBRARY_TO_NON_SOLIB = + new Function<LibraryToLink, Artifact>() { + @Override + public Artifact apply(LibraryToLink input) { + return input.getOriginalLibraryArtifact(); + } + }; + + public static Iterable<Artifact> toNonSolibArtifacts(Iterable<LibraryToLink> libraries) { + return Iterables.transform(libraries, LIBRARY_TO_NON_SOLIB); + } + + /** + * Returns the linker input artifacts from a collection of {@link LinkerInput} objects. + */ + public static Iterable<Artifact> toLibraryArtifacts(Iterable<? extends LinkerInput> artifacts) { + return Iterables.transform(artifacts, new Function<LinkerInput, Artifact>() { + @Override + public Artifact apply(LinkerInput input) { + return input.getArtifact(); + } + }); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkingMode.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkingMode.java new file mode 100644 index 0000000000..8018108bea --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkingMode.java @@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +/** + * This class represents the different linking modes. + */ +public enum LinkingMode { + + /** + * Everything is linked statically; e.g. {@code gcc -static x.o libfoo.a + * libbar.a -lm}. Specified by {@code -static} in linkopts. + */ + FULLY_STATIC, + + /** + * Link binaries statically except for system libraries + * e.g. {@code gcc x.o libfoo.a libbar.a -lm}. Specified by {@code linkstatic=1}. + * + * <p>This mode applies to executables. + */ + MOSTLY_STATIC, + + /** + * Same as MOSTLY_STATIC, but for shared libraries. + */ + MOSTLY_STATIC_LIBRARIES, + + /** + * All libraries are linked dynamically (if a dynamic version is available), + * e.g. {@code gcc x.o libfoo.so libbar.so -lm}. Specified by {@code + * linkstatic=0}. + */ + DYNAMIC; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LipoContextProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LipoContextProvider.java new file mode 100644 index 0000000000..a9ffea8f6b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LipoContextProvider.java @@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.Map; + +/** + * Provides LIPO context information to the LIPO-enabled target configuration. + * + * <p>This is a rollup of the data collected in the LIPO context collector configuration. + * Each target in the LIPO context collector configuration has a {@link TransitiveLipoInfoProvider} + * which is used to transitively collect the data, then the {@code cc_binary} that is referred to + * in {@code --lipo_context} puts the collected data into {@link LipoContextProvider}, of which + * there is only one in any given build. + */ +@Immutable +public final class LipoContextProvider implements TransitiveInfoProvider { + + private final CppCompilationContext cppCompilationContext; + + private final ImmutableMap<Artifact, IncludeScannable> includeScannables; + public LipoContextProvider(CppCompilationContext cppCompilationContext, + Map<Artifact, IncludeScannable> scannables) { + this.cppCompilationContext = cppCompilationContext; + this.includeScannables = ImmutableMap.copyOf(scannables); + } + + /** + * Returns merged compilation context for the whole LIPO subtree. + */ + public CppCompilationContext getLipoContext() { + return cppCompilationContext; + } + + /** + * Returns the map from source artifact to the include scannable object representing + * the corresponding FDO source input file. + */ + public ImmutableMap<Artifact, IncludeScannable> getIncludeScannables() { + return includeScannables; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalGccStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalGccStrategy.java new file mode 100644 index 0000000000..80ee23d70b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalGccStrategy.java @@ -0,0 +1,96 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BaseSpawn; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.common.options.OptionsClassProvider; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * Run gcc locally by delegating to spawn. + */ +@ExecutionStrategy(name = { "local" }, + contextType = CppCompileActionContext.class) +public class LocalGccStrategy implements CppCompileActionContext { + private static final Reply CANNED_REPLY = new Reply() { + @Override + public byte[] getContents() { + throw new IllegalStateException("Remotely computed data requested for local action"); + } + }; + + public LocalGccStrategy(OptionsClassProvider options) { + } + + @Override + public String strategyLocality() { + return "local"; + } + + public static void updateEnv(CppCompileAction action, Map<String, String> env) { + // We cannot locally execute an action that does not expect to output a .d file, since we would + // have no way to tell what files that it included were used during compilation. + env.put("INTERCEPT_LOCALLY_EXECUTABLE", action.getDotdFile().artifact() == null ? "0" : "1"); + } + + @Override + public boolean needsIncludeScanning() { + return false; + } + + @Override + public Collection<? extends ActionInput> findAdditionalInputs(CppCompileAction action, + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException { + return ImmutableList.of(); + } + + @Override + public CppCompileActionContext.Reply execWithReply( + CppCompileAction action, ActionExecutionContext actionExecutionContext) + throws ExecException, InterruptedException { + Map<String, String> env = new HashMap<>(); + env.putAll(action.getEnvironment()); + updateEnv(action, env); + actionExecutionContext.getExecutor().getSpawnActionContext(action.getMnemonic()) + .exec(new BaseSpawn.Local(action.getArgv(), env, action), + actionExecutionContext); + return null; + } + + @Override + public ResourceSet estimateResourceConsumption(CppCompileAction action) { + return action.estimateResourceConsumptionLocal(); + } + + @Override + public Collection<Artifact> getScannedIncludeFiles( + CppCompileAction action, ActionExecutionContext actionExecutionContext) { + return ImmutableList.of(); + } + + @Override + public Reply getReplyFromException(ExecException e, CppCompileAction action) { + return CANNED_REPLY; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalLinkStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalLinkStrategy.java new file mode 100644 index 0000000000..3e7c863f42 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalLinkStrategy.java @@ -0,0 +1,62 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.BaseSpawn; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; + +import java.util.List; + +/** + * A link strategy that runs the linking step on the local host. + * + * <p>The set of input files necessary to successfully complete the link is the middleman-expanded + * set of the action's dependency inputs (which includes crosstool and libc dependencies, as + * defined by {@link com.google.devtools.build.lib.rules.cpp.CppHelper#getCrosstoolInputsForLink + * CppHelper.getCrosstoolInputsForLink}). + */ +@ExecutionStrategy(contextType = CppLinkActionContext.class, name = { "local" }) +public final class LocalLinkStrategy extends LinkStrategy { + + public LocalLinkStrategy() { + } + + @Override + public void exec(CppLinkAction action, ActionExecutionContext actionExecutionContext) + throws ExecException, ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + List<String> argv = + action.prepareCommandLine(executor.getExecRoot(), null); + executor.getSpawnActionContext(action.getMnemonic()).exec( + new BaseSpawn.Local(argv, ImmutableMap.<String, String>of(), action), + actionExecutionContext); + } + + @Override + public String linkStrategyName() { + return "local"; + } + + @Override + public ResourceSet estimateResourceConsumption(CppLinkAction action) { + return action.estimateResourceConsumptionLocal(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/RemoteIncludeExtractor.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/RemoteIncludeExtractor.java new file mode 100644 index 0000000000..87a07123c6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/RemoteIncludeExtractor.java @@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.rules.cpp.IncludeParser.Inclusion; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.IOException; +import java.util.Collection; + +/** Parses a single file for its (direct) includes, possibly using a remote service. */ +public interface RemoteIncludeExtractor extends ActionContext { + /** Result of checking if this object should be used to parse a given file. */ + interface RemoteParseData { + boolean shouldParseRemotely(); + } + + /** + * Returns whether to use this object to parse the given file for includes. The returned data + * should be passed to {@link #extractInclusions} to direct its behavior. + */ + RemoteParseData shouldParseRemotely(Path file); + + /** + * Extracts all inclusions from a given source file, possibly using a remote service. + * + * @param file the file from which to parse and extract inclusions. + * @param actionExecutionContext services in the scope of the action. Like the Err/Out stream + * outputs. + * @param remoteParseData the returned value of {@link #shouldParseRemotely}. + * @return a collection of inclusions, normalized to the cache + */ + public Collection<Inclusion> extractInclusions(Artifact file, + ActionExecutionContext actionExecutionContext, RemoteParseData remoteParseData) + throws IOException, InterruptedException; + +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/SolibSymlinkAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/SolibSymlinkAction.java new file mode 100644 index 0000000000..120ba86a34 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/SolibSymlinkAction.java @@ -0,0 +1,234 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Actions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; + +/** + * Creates mangled symlinks in the solib directory for all shared libraries. + * Libraries that have a potential to contain SONAME field rely on the mangled + * symlink to the parent directory instead. + * + * Such symlinks are used by the linker to ensure that all rpath entries can be + * specified relative to the $ORIGIN. + */ +public final class SolibSymlinkAction extends AbstractAction { + + private final Artifact library; + private final Path target; + private final Artifact symlink; + + private SolibSymlinkAction(ActionOwner owner, Artifact library, Artifact symlink) { + super(owner, ImmutableList.of(library), ImmutableList.of(symlink)); + + Preconditions.checkArgument(Link.SHARED_LIBRARY_FILETYPES.matches(library.getFilename())); + this.library = Preconditions.checkNotNull(library); + this.symlink = Preconditions.checkNotNull(symlink); + this.target = library.getPath(); + } + + @Override + protected void deleteOutputs(Path execRoot) throws IOException { + // Do not delete outputs if action does not intend to do anything. + if (target != null) { + super.deleteOutputs(execRoot); + } + } + + @Override + public void execute( + ActionExecutionContext actionExecutionContext) throws ActionExecutionException { + Path mangledPath = symlink.getPath(); + try { + FileSystemUtils.createDirectoryAndParents(mangledPath.getParentDirectory()); + mangledPath.createSymbolicLink(target); + } catch (IOException e) { + throw new ActionExecutionException("failed to create _solib symbolic link '" + + symlink.prettyPrint() + "' to target '" + target + "'", e, this, false); + } + } + + @Override + public Artifact getPrimaryInput() { + return library; + } + + @Override + public Artifact getPrimaryOutput() { + return symlink; + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return new ResourceSet(/*memoryMb=*/0, /*cpuUsage=*/0, /*ioUsage=*/0.0); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addPath(symlink.getPath()); + if (target != null) { + f.addPath(target); + } + return f.hexDigestAndReset(); + } + + @Override + public String getMnemonic() { return "SolibSymlink"; } + + @Override + public String describeStrategy(Executor executor) { + return "local"; + } + + @Override + protected String getRawProgressMessage() { return null; } + + /** + * Replaces shared library artifact with mangled symlink and creates related + * symlink action. For artifacts that should retain filename (e.g. libraries + * with SONAME tag), link is created to the parent directory instead. + * + * This action is performed to minimize number of -rpath entries used during + * linking process (by essentially "collecting" as many shared libraries as + * possible in the single directory), since we will be paying quadratic price + * for each additional entry on the -rpath. + * + * @param ruleContext rule context, that requested symlink. + * @param library Shared library artifact that needs to be mangled. + * @param preserveName whether to preserve the name of the library + * @param prefixConsumer whether to prefix the output artifact name with the label of the + * consumer + * @return mangled symlink artifact. + */ + public static LibraryToLink getDynamicLibrarySymlink(final RuleContext ruleContext, + final Artifact library, + boolean preserveName, + boolean prefixConsumer, + BuildConfiguration configuration) { + PathFragment mangledName = getMangledName( + ruleContext, library.getRootRelativePath(), preserveName, prefixConsumer, + configuration.getFragment(CppConfiguration.class)); + return getDynamicLibrarySymlinkInternal(ruleContext, library, mangledName, configuration); + } + + /** + * Version of {@link #getDynamicLibrarySymlink} for the special case of C++ runtime libraries. + * These are handled differently than other libraries: neither their names nor directories are + * mangled, i.e. libstdc++.so.6 is symlinked from _solib_[arch]/libstdc++.so.6 + */ + public static LibraryToLink getCppRuntimeSymlink(RuleContext ruleContext, Artifact library, + String solibDirOverride, BuildConfiguration configuration) { + PathFragment solibDir = new PathFragment(solibDirOverride != null + ? solibDirOverride + : configuration.getFragment(CppConfiguration.class).getSolibDirectory()); + PathFragment symlinkName = solibDir.getRelative(library.getRootRelativePath().getBaseName()); + return getDynamicLibrarySymlinkInternal(ruleContext, library, symlinkName, configuration); + } + + /** + * Internal implementation that takes a pre-determined symlink name; supports both the + * generic {@link #getDynamicLibrarySymlink} and the specialized {@link #getCppRuntimeSymlink}. + */ + private static LibraryToLink getDynamicLibrarySymlinkInternal(RuleContext ruleContext, + Artifact library, PathFragment symlinkName, BuildConfiguration configuration) { + Preconditions.checkArgument(Link.SHARED_LIBRARY_FILETYPES.matches(library.getFilename())); + Preconditions.checkArgument(!library.getRootRelativePath().getSegment(0).startsWith("_solib_")); + + // Ignore libraries that are already represented by the symlinks. + Root root = configuration.getBinDirectory(); + Artifact symlink = ruleContext.getAnalysisEnvironment().getDerivedArtifact(symlinkName, root); + ruleContext.registerAction( + new SolibSymlinkAction(ruleContext.getActionOwner(), library, symlink)); + return LinkerInputs.solibLibraryToLink(symlink, library); + } + + /** + * Returns the name of the symlink that will be created for a library, given + * its name. + * + * @param ruleContext rule context that requests symlink + * @param libraryPath the root-relative path of the library + * @param preserveName true if filename should be preserved + * @param prefixConsumer true if the result should be prefixed with the label of the consumer + * @returns root relative path name + */ + public static PathFragment getMangledName(RuleContext ruleContext, + PathFragment libraryPath, + boolean preserveName, + boolean prefixConsumer, + CppConfiguration cppConfiguration) { + String escapedRulePath = Actions.escapedPath( + "_" + ruleContext.getLabel()); + String soname = getDynamicLibrarySoname(libraryPath, preserveName); + PathFragment solibDir = new PathFragment(cppConfiguration.getSolibDirectory()); + if (preserveName) { + String escapedLibraryPath = + Actions.escapedPath("_" + libraryPath.getParentDirectory().getPathString()); + PathFragment mangledDir = solibDir.getRelative(prefixConsumer + ? escapedRulePath + "__" + escapedLibraryPath + : escapedLibraryPath); + return mangledDir.getRelative(soname); + } else { + return solibDir.getRelative(prefixConsumer + ? escapedRulePath + "__" + soname + : soname); + } + } + + /** + * Compute the SONAME to use for a dynamic library. This name is basically the + * name of the shared library in its final symlinked location. + * + * @param libraryPath name of the shared library that needs to be mangled + * @param preserveName true if filename should be preserved, false - mangled + * @return soname to embed in the dynamic library + */ + public static String getDynamicLibrarySoname(PathFragment libraryPath, + boolean preserveName) { + String mangledName; + if (preserveName) { + mangledName = libraryPath.getBaseName(); + } else { + mangledName = "lib" + Actions.escapedPath(libraryPath.getPathString()); + } + return mangledName; + } + + @Override + public boolean shouldReportPathPrefixConflict(Action action) { + return false; // Always ignore path prefix conflict for the SolibSymlinkAction. + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/TransitiveLipoInfoProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/TransitiveLipoInfoProvider.java new file mode 100644 index 0000000000..40941249a2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/TransitiveLipoInfoProvider.java @@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A target that can contribute profiling information to LIPO C++ compilations. + * + * <p>This is used in the LIPO context collector tree to collect data from the transitive + * closure of the :lipo_context_collector target. It is eventually passed to the configured + * targets in the target configuration through {@link LipoContextProvider}. + */ +@Immutable +public final class TransitiveLipoInfoProvider implements TransitiveInfoProvider { + public static final TransitiveLipoInfoProvider EMPTY = + new TransitiveLipoInfoProvider( + NestedSetBuilder.<IncludeScannable>emptySet(Order.STABLE_ORDER)); + + private final NestedSet<IncludeScannable> includeScannables; + + public TransitiveLipoInfoProvider(NestedSet<IncludeScannable> includeScannables) { + this.includeScannables = includeScannables; + } + + /** + * Returns the include scannables in the transitive closure. + * + * <p>This is used for constructing the path fragment -> include scannable map in the + * LIPO-enabled target configuration. + */ + public NestedSet<IncludeScannable> getTransitiveIncludeScannables() { + return includeScannables; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/WriteBuildInfoHeaderAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/WriteBuildInfoHeaderAction.java new file mode 100644 index 0000000000..58b33309b6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/WriteBuildInfoHeaderAction.java @@ -0,0 +1,194 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.analysis.BuildInfoHelper; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * An action that creates a C++ header containing the build information in the + * form of #define directives. + */ +public final class WriteBuildInfoHeaderAction extends AbstractFileWriteAction { + private static final String GUID = "b0798174-1352-4a54-854a-9785aaea491b"; + + private final ImmutableList<Artifact> valueArtifacts; + + private final boolean writeVolatileInfo; + private final boolean writeStableInfo; + + /** + * Creates an action that writes a C++ header with the build information. + * + * <p>It reads the set of build info keys from an action context that is usually contributed + * to Bazel by the workspace status module, and the value associated with said keys from the + * workspace status files (stable and volatile) written by the workspace status action. + * + * <p>Without input artifacts this action uses redacted build information. + * @param inputs Artifacts that contain build information, or an empty + * collection to use redacted build information + * @param output the C++ header Artifact created by this action + * @param writeVolatileInfo whether to write the volatile part of the build + * information to the generated header + * @param writeStableInfo whether to write the non-volatile part of the + * build information to the generated header + */ + public WriteBuildInfoHeaderAction(Collection<Artifact> inputs, + Artifact output, boolean writeVolatileInfo, boolean writeStableInfo) { + super(BuildInfoHelper.BUILD_INFO_ACTION_OWNER, + inputs, output, /*makeExecutable=*/false); + valueArtifacts = ImmutableList.copyOf(inputs); + if (!inputs.isEmpty()) { + // With non-empty inputs we should not generate both volatile and non-volatile data + // in the same header file. + Preconditions.checkState(writeVolatileInfo ^ writeStableInfo); + } + Preconditions.checkState( + output.isConstantMetadata() == (writeVolatileInfo && !inputs.isEmpty())); + + this.writeVolatileInfo = writeVolatileInfo; + this.writeStableInfo = writeStableInfo; + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) + throws IOException { + WorkspaceStatusAction.Context context = + executor.getContext(WorkspaceStatusAction.Context.class); + + final Map<String, WorkspaceStatusAction.Key> keys = new LinkedHashMap<>(); + if (writeVolatileInfo) { + keys.putAll(context.getVolatileKeys()); + } + + if (writeStableInfo) { + keys.putAll(context.getStableKeys()); + } + + final Map<String, String> values = new LinkedHashMap<>(); + for (Artifact valueFile : valueArtifacts) { + values.putAll(WorkspaceStatusAction.parseValues(valueFile.getPath())); + } + + final boolean redacted = valueArtifacts.isEmpty(); + + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + Writer writer = new OutputStreamWriter(out, UTF_8); + + for (Map.Entry<String, WorkspaceStatusAction.Key> key : keys.entrySet()) { + if (!key.getValue().isInLanguage("C++")) { + continue; + } + + String value = redacted ? key.getValue().getRedactedValue() + : values.containsKey(key.getKey()) ? values.get(key.getKey()) + : key.getValue().getDefaultValue(); + + switch (key.getValue().getType()) { + case VERBATIM: + case INTEGER: + break; + + case STRING: + value = quote(value); + break; + + default: + throw new IllegalStateException(); + } + define(writer, key.getKey(), value); + + } + writer.flush(); + } + }; + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addBoolean(writeStableInfo); + f.addBoolean(writeVolatileInfo); + return f.hexDigestAndReset(); + } + + @Override + public boolean executeUnconditionally() { + // Note: isVolatile must return true if executeUnconditionally can ever return true + // for this instance. + return isUnconditional(); + } + + @Override + public boolean isVolatile() { + return isUnconditional(); + } + + private boolean isUnconditional() { + // Because of special handling in the MetadataHandler, changed volatile build + // information does not trigger relinking of all libraries that have + // linkstamps. But we do want to regenerate the header in case libraries are + // relinked because of other reasons. + // Without inputs the contents of the header do not change, so there is no + // point in executing the action again in that case. + return writeVolatileInfo && !Iterables.isEmpty(getInputs()); + } + + /** + * Quote a string with double quotes. + */ + private String quote(String string) { + // TODO(bazel-team): This is doesn't really work if the string contains quotes. Or a newline. + // Or a backslash. Or anything unusual, really. + return "\"" + string + "\""; + } + + /** + * Write a preprocessor define directive to a Writer. + */ + private void define(Writer writer, String name, String value) throws IOException { + writer.write("#define "); + writer.write(name); + writer.write(' '); + writer.write(value); + writer.write('\n'); + } + + @Override + protected String getRawProgressMessage() { + return null; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ActionListener.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ActionListener.java new file mode 100644 index 0000000000..f3b302f13e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ActionListener.java @@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.extra; + +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.ImmutableSortedKeyListMultimap; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * Implementation for the 'action_listener' rule. + */ +public final class ActionListener implements RuleConfiguredTargetFactory { + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + // This rule doesn't produce any output when listed as a build target. + // Only when used via the --experimental_action_listener flag, + // this rule instructs the build system to add additional outputs. + + List<ExtraActionSpec> extraActions; + + Multimap<String, ExtraActionSpec> extraActionMap; + + Set<String> mnemonics = Sets.newHashSet( + ruleContext.attributes().get("mnemonics", Type.STRING_LIST)); + extraActions = retrieveAndValidateExtraActions(ruleContext); + ImmutableSortedKeyListMultimap.Builder<String, ExtraActionSpec> + extraActionMapBuilder = ImmutableSortedKeyListMultimap.builder(); + for (String mnemonic : mnemonics) { + extraActionMapBuilder.putAll(mnemonic, extraActions); + } + extraActionMap = extraActionMapBuilder.build(); + return new RuleConfiguredTargetBuilder(ruleContext) + .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY)) + .add(ExtraActionMapProvider.class, new ExtraActionMapProvider(extraActionMap)) + .build(); + } + + /** + * Loads the targets listed in the 'extra_actions' attribute of this rule. + * Validates these targets to be extra_actions indeed. And checks if the + * blaze version number is in the range of the blaze_version restrictions on the rule. + */ + private List<ExtraActionSpec> retrieveAndValidateExtraActions(RuleContext ruleContext) { + List<ExtraActionSpec> extraActions = new ArrayList<>(); + for (TransitiveInfoCollection prerequisite : + ruleContext.getPrerequisites("extra_actions", Mode.TARGET)) { + ExtraActionSpec spec = prerequisite.getProvider(ExtraActionSpec.class); + if (spec == null) { + ruleContext.attributeError("extra_actions", String.format("target %s is not an " + + "extra_action rule", prerequisite.getLabel().toString())); + } else { + extraActions.add(spec); + } + } + if (extraActions.size() == 0) { + ruleContext.attributeWarning("extra_actions", + "No extra_action is specified for this version of blaze."); + } + return extraActions; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraAction.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraAction.java new file mode 100644 index 0000000000..2b53a1f1b1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraAction.java @@ -0,0 +1,246 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.extra; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactResolver; +import com.google.devtools.build.lib.actions.DelegateSpawn; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Action used by extra_action rules to create an action that shadows an existing action. Runs a + * command-line using {@link SpawnActionContext} for executions. + */ +public final class ExtraAction extends SpawnAction { + private final Action shadowedAction; + private final boolean createDummyOutput; + private final Artifact extraActionInfoFile; + private final ImmutableMap<PathFragment, Artifact> runfilesManifests; + private final ImmutableSet<Artifact> extraActionInputs; + private boolean inputsKnown; + + public ExtraAction(ActionOwner owner, + ImmutableSet<Artifact> extraActionInputs, + Map<PathFragment, Artifact> runfilesManifests, + Artifact extraActionInfoFile, + Collection<Artifact> outputs, + Action shadowedAction, + boolean createDummyOutput, + CommandLine argv, + Map<String, String> environment, + String progressMessage, + String mnemonic) { + super(owner, + createInputs(shadowedAction.getInputs(), extraActionInputs), + outputs, + AbstractAction.DEFAULT_RESOURCE_SET, + argv, environment, progressMessage, mnemonic); + this.extraActionInfoFile = extraActionInfoFile; + this.shadowedAction = shadowedAction; + this.runfilesManifests = ImmutableMap.copyOf(runfilesManifests); + this.createDummyOutput = createDummyOutput; + + this.extraActionInputs = extraActionInputs; + inputsKnown = shadowedAction.inputsKnown(); + if (createDummyOutput) { + // extra action file & dummy file + Preconditions.checkArgument(outputs.size() == 2); + } + } + + @Override + public boolean discoversInputs() { + return shadowedAction.discoversInputs(); + } + + @Override + public void discoverInputs(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Preconditions.checkState(discoversInputs(), this); + if (getContext(actionExecutionContext.getExecutor()).isRemotable(getMnemonic(), + isRemotable())) { + // If we're running remotely, we need to update our inputs to take account of any additional + // inputs the shadowed action may need to do its work. + if (shadowedAction.discoversInputs() && shadowedAction instanceof AbstractAction) { + updateInputs( + ((AbstractAction) shadowedAction).getInputFilesForExtraAction(actionExecutionContext)); + } + } + } + + @Override + public boolean inputsKnown() { + return inputsKnown; + } + + private static NestedSet<Artifact> createInputs( + Iterable<Artifact> shadowedActionInputs, ImmutableSet<Artifact> extraActionInputs) { + NestedSetBuilder<Artifact> result = new NestedSetBuilder<>(Order.STABLE_ORDER); + if (shadowedActionInputs instanceof NestedSet) { + result.addTransitive((NestedSet<Artifact>) shadowedActionInputs); + } else { + result.addAll(shadowedActionInputs); + } + return result.addAll(extraActionInputs).build(); + } + + private void updateInputs(Iterable<Artifact> shadowedActionInputs) { + synchronized (this) { + setInputs(createInputs(shadowedActionInputs, extraActionInputs)); + inputsKnown = true; + } + } + + @Override + public void updateInputsFromCache(ArtifactResolver artifactResolver, + Collection<PathFragment> inputPaths) { + // We update the inputs directly from the shadowed action. + Set<PathFragment> extraActionPathFragments = + ImmutableSet.copyOf(Artifact.asPathFragments(extraActionInputs)); + shadowedAction.updateInputsFromCache(artifactResolver, + Collections2.filter(inputPaths, Predicates.in(extraActionPathFragments))); + Preconditions.checkState(shadowedAction.inputsKnown(), "%s %s", this, shadowedAction); + updateInputs(shadowedAction.getInputs()); + } + + /** + * @InheritDoc + * + * This method calls in to {@link AbstractAction#getInputFilesForExtraAction} and + * {@link Action#getExtraActionInfo} of the action being shadowed from the thread executing this + * ExtraAction. It assumes these methods are safe to call from a different thread than the thread + * responsible for the execution of the action being shadowed. + */ + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + // PHASE 1: generate .xa file containing protocol buffer describing + // the action being shadowed + + // We call the getExtraActionInfo command only at execution time + // so actions can store information only known at execution time into the + // protocol buffer. + ExtraActionInfo info = shadowedAction.getExtraActionInfo().build(); + try (OutputStream out = extraActionInfoFile.getPath().getOutputStream()) { + info.writeTo(out); + } catch (IOException e) { + throw new ActionExecutionException(e.getMessage(), e, this, false); + } + Executor executor = actionExecutionContext.getExecutor(); + + // PHASE 2: execution of extra_action. + + if (getContext(executor).isRemotable(getMnemonic(), isRemotable())) { + try { + getContext(executor).exec(getExtraActionSpawn(), actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException(this); + } + } else { + super.execute(actionExecutionContext); + } + + // PHASE 3: create dummy output. + // If the user didn't specify output, we need to create dummy output + // to make blaze schedule this action. + if (createDummyOutput) { + for (Artifact output : getOutputs()) { + try { + FileSystemUtils.touchFile(output.getPath()); + } catch (IOException e) { + throw new ActionExecutionException(e.getMessage(), e, this, false); + } + } + } + synchronized (this) { + inputsKnown = true; + } + } + + /** + * The spawn command for ExtraAction needs to be slightly modified from + * regular SpawnActions: + * -the extraActionInfo file needs to be added to the list of inputs. + * -the extraActionInfo file that is an output file of this task is created + * before the SpawnAction so should not be listed as one of its outputs. + */ + // TODO(bazel-team): Add more tests that execute this code path! + private Spawn getExtraActionSpawn() { + final Spawn base = super.getSpawn(); + return new DelegateSpawn(base) { + @Override public Iterable<? extends ActionInput> getInputFiles() { + return Iterables.concat(base.getInputFiles(), ImmutableSet.of(extraActionInfoFile)); + } + + @Override public List<? extends ActionInput> getOutputFiles() { + return Lists.newArrayList( + Iterables.filter(getOutputs(), new Predicate<Artifact>() { + @Override + public boolean apply(Artifact item) { + return item != extraActionInfoFile; + } + })); + } + + @Override public ImmutableMap<PathFragment, Artifact> getRunfilesManifests() { + ImmutableMap.Builder<PathFragment, Artifact> builder = ImmutableMap.builder(); + builder.putAll(super.getRunfilesManifests()); + builder.putAll(runfilesManifests); + return builder.build(); + } + + @Override public String getMnemonic() { return ExtraAction.this.getMnemonic(); } + }; + } + + /** + * Returns the action this extra action is 'shadowing'. + */ + public Action getShadowedAction() { + return shadowedAction; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionFactory.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionFactory.java new file mode 100644 index 0000000000..8040ee0c03 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionFactory.java @@ -0,0 +1,91 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.extra; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.CommandHelper; +import com.google.devtools.build.lib.analysis.ConfigurationMakeVariableContext; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.MakeVariableExpander; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.List; + +/** + * Factory for 'extra_action'. + */ +public final class ExtraActionFactory implements RuleConfiguredTargetFactory { + @Override + public ConfiguredTarget create(RuleContext context) { + // This rule doesn't produce any output when listed as a build target. + // Only when used via the --experimental_action_listener flag, + // this rule instructs the build system to add additional outputs. + List<Artifact> resolvedData = Lists.newArrayList(); + + Iterable<FilesToRunProvider> tools = + context.getPrerequisites("tools", Mode.HOST, FilesToRunProvider.class); + CommandHelper commandHelper = new CommandHelper( + context, tools, ImmutableMap.<Label, Iterable<Artifact>>of()); + + resolvedData.addAll(context.getPrerequisiteArtifacts("data", Mode.DATA).list()); + List<String>outputTemplates = + context.attributes().get("out_templates", Type.STRING_LIST); + + String command = commandHelper.resolveCommandAndExpandLabels(false, true); + // This is a bit of a hack. We want to run the MakeVariableExpander first, so we expand $ on + // variables that are expanded below with $$, which gets reverted to $ by the + // MakeVariableExpander. This allows us to expand package-specific make variables in the + // package where the extra action is defined, and then later replace the owner-specific make + // variables when the extra action is instantiated. + command = command.replace("$(EXTRA_ACTION_FILE)", "$$(EXTRA_ACTION_FILE)"); + command = command.replace("$(ACTION_ID)", "$$(ACTION_ID)"); + command = command.replace("$(OWNER_LABEL_DIGEST)", "$$(OWNER_LABEL_DIGEST)"); + command = command.replace("$(output ", "$$(output "); + try { + command = MakeVariableExpander.expand( + command, new ConfigurationMakeVariableContext( + context.getTarget().getPackage(), context.getConfiguration())); + } catch (MakeVariableExpander.ExpansionException e) { + context.ruleError(String.format("Unable to expand make variables: %s", + e.getMessage())); + } + + boolean requiresActionOutput = + context.attributes().get("requires_action_output", Type.BOOLEAN); + + ExtraActionSpec spec = new ExtraActionSpec( + commandHelper.getResolvedTools(), + commandHelper.getRemoteRunfileManifestMap(), + resolvedData, + outputTemplates, + command, + context.getLabel(), + requiresActionOutput); + + return new RuleConfiguredTargetBuilder(context) + .addProvider(ExtraActionSpec.class, spec) + .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY)) + .build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionMapProvider.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionMapProvider.java new file mode 100644 index 0000000000..ffeebe0c03 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionMapProvider.java @@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.extra; + +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Provides an action type -> set of extra actions to run map. + */ +@Immutable +public final class ExtraActionMapProvider implements TransitiveInfoProvider { + private final ImmutableMultimap<String, ExtraActionSpec> extraActionMap; + + public ExtraActionMapProvider(Multimap<String, ExtraActionSpec> extraActionMap) { + this.extraActionMap = ImmutableMultimap.copyOf(extraActionMap); + } + + /** + * Returns the extra action map. + */ + public ImmutableMultimap<String, ExtraActionSpec> getExtraActionMap() { + return extraActionMap; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionSpec.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionSpec.java new file mode 100644 index 0000000000..40a063eddd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionSpec.java @@ -0,0 +1,220 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.extra; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.CommandHelper; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; + +/** + * The specification for a particular extra action type. + */ +@Immutable +public final class ExtraActionSpec implements TransitiveInfoProvider { + private final ImmutableList<Artifact> resolvedTools; + private final ImmutableMap<PathFragment, Artifact> manifests; + private final ImmutableList<Artifact> resolvedData; + private final ImmutableList<String> outputTemplates; + private final String command; + private final boolean requiresActionOutput; + private final Label label; + + ExtraActionSpec( + Iterable<Artifact> resolvedTools, + Map<PathFragment, Artifact> manifests, + Iterable<Artifact> resolvedData, + Iterable<String> outputTemplates, + String command, + Label label, + boolean requiresActionOutput) { + this.resolvedTools = ImmutableList.copyOf(resolvedTools); + this.manifests = ImmutableMap.copyOf(manifests); + this.resolvedData = ImmutableList.copyOf(resolvedData); + this.outputTemplates = ImmutableList.copyOf(outputTemplates); + this.command = command; + this.label = label; + this.requiresActionOutput = requiresActionOutput; + } + + public Label getLabel() { + return label; + } + + /** + * Adds an extra_action to the action graph based on the action to shadow. + */ + public Collection<Artifact> addExtraAction(RuleContext owningRule, + Action actionToShadow) { + Collection<Artifact> extraActionOutputs = new LinkedHashSet<>(); + ImmutableSet.Builder<Artifact> extraActionInputs = ImmutableSet.builder(); + + ActionOwner owner = actionToShadow.getOwner(); + Label ownerLabel = owner.getLabel(); + if (requiresActionOutput) { + extraActionInputs.addAll(actionToShadow.getOutputs()); + } + extraActionInputs.addAll(resolvedTools); + extraActionInputs.addAll(resolvedData); + + boolean createDummyOutput = false; + + for (String outputTemplate : outputTemplates) { + // We create output for the extra_action based on the 'out_template' attribute. + // See {link #getExtraActionOutputArtifact} for supported variables. + extraActionOutputs.add(getExtraActionOutputArtifact(owningRule, actionToShadow, + owner, outputTemplate)); + } + // extra_action has no output, we need to create some dummy output to keep the build up-to-date. + if (extraActionOutputs.size() == 0) { + createDummyOutput = true; + extraActionOutputs.add(getExtraActionOutputArtifact(owningRule, actionToShadow, + owner, "$(ACTION_ID).dummy")); + } + + // We generate a file containing a protocol buffer describing the action that is being shadowed. + // It is up to each action being shadowed to decide what contents to store here. + Artifact extraActionInfoFile = getExtraActionOutputArtifact(owningRule, actionToShadow, + owner, "$(ACTION_ID).xa"); + extraActionOutputs.add(extraActionInfoFile); + + // Expand extra_action specific variables from the provided command-line. + // See {@link #createExpandedCommand} for list of supported variables. + String command = createExpandedCommand(owningRule, actionToShadow, owner, extraActionInfoFile); + + Map<String, String> env = owningRule.getConfiguration().getDefaultShellEnvironment(); + + List<String> argv = CommandHelper.buildCommandLine(owningRule, + command, extraActionInputs, ".extra_action_script.sh"); + + String commandMessage = String.format("Executing extra_action %s on %s", label, ownerLabel); + owningRule.registerAction(new ExtraAction( + actionToShadow.getOwner(), + extraActionInputs.build(), + manifests, + extraActionInfoFile, + extraActionOutputs, + actionToShadow, + createDummyOutput, + CommandLine.of(argv, false), + env, + commandMessage, + label.getName())); + + return extraActionOutputs; + } + + /** + * Expand extra_action specific variables: + * $(EXTRA_ACTION_FILE): expands to a path of the file containing a protocol buffer + * describing the action being shadowed. + * $(output <out_template>): expands the output template to the execPath of the file. + * e.g. $(output $(ACTION_ID).out) -> + * <build_path>/extra_actions/bar/baz/devtools/build/test_A41234.out + */ + private String createExpandedCommand(RuleContext owningRule, + Action action, ActionOwner owner, Artifact extraActionInfoFile) { + String realCommand = command.replace( + "$(EXTRA_ACTION_FILE)", extraActionInfoFile.getExecPathString()); + + for (String outputTemplate : outputTemplates) { + String outFile = getExtraActionOutputArtifact(owningRule, action, owner, outputTemplate) + .getExecPathString(); + realCommand = realCommand.replace("$(output " + outputTemplate + ")", outFile); + } + return realCommand; + } + + /** + * Creates an output artifact for the extra_action based on the output_template. + * The path will be in the following form: + * <output dir>/<target-configuration-specific-path>/extra_actions/<extra_action_label>/ + + * <configured_target_label>/<expanded_template> + * + * The template can use the following variables: + * $(ACTION_ID): a unique id for the extra_action. + * + * Sample: + * extra_action: foo/bar:extra + * template: $(ACTION_ID).analysis + * target: foo/bar:main + * expands to: output/configuration/extra_actions/\ + * foo/bar/extra/foo/bar/4683026f7ac1dd1a873ccc8c3d764132.analysis + */ + private Artifact getExtraActionOutputArtifact(RuleContext owningRule, Action action, + ActionOwner owner, String template) { + String actionId = getActionId(owner, action); + + template = template.replace("$(ACTION_ID)", actionId); + template = template.replace("$(OWNER_LABEL_DIGEST)", getOwnerDigest(owner)); + + PathFragment rootRelativePath = getRootRelativePath(template, owner); + return owningRule.getAnalysisEnvironment().getDerivedArtifact(rootRelativePath, + owningRule.getConfiguration().getOutputDirectory()); + } + + private PathFragment getRootRelativePath(String template, ActionOwner owner) { + PathFragment extraActionPackageFragment = label.getPackageFragment(); + PathFragment extraActionPrefix = extraActionPackageFragment.getRelative(label.getName()); + + PathFragment ownerFragment = owner.getLabel().getPackageFragment(); + return new PathFragment("extra_actions").getRelative(extraActionPrefix) + .getRelative(ownerFragment).getRelative(template); + } + + /** + * Calculates a digest representing the owner label. We use the digest instead of the + * original value as the original value might lead to a filename that is too long. + * By using a digest, tools can deterministically find all extra_action outputs for a given + * target, without having to open every file in the package. + */ + private static String getOwnerDigest(ActionOwner owner) { + Fingerprint f = new Fingerprint(); + f.addString(owner.getLabel().toString()); + return f.hexDigestAndReset(); + } + + /** + * Creates a unique id for the action shadowed by this extra_action. + * + * We need to have a unique id for the extra_action to use. We build this + * from the owner's label and the shadowed action id (which is only + * guaranteed to be unique per target). Together with the subfolder + * matching the original target's package name, we believe this is enough + * of a uniqueness guarantee. + */ + @VisibleForTesting + public static String getActionId(ActionOwner owner, Action action) { + Fingerprint f = new Fingerprint(); + f.addString(owner.getLabel().toString()); + f.addString(action.getKey()); + return f.hexDigestAndReset(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/filegroup/Filegroup.java b/src/main/java/com/google/devtools/build/lib/rules/filegroup/Filegroup.java new file mode 100644 index 0000000000..cb297d8d0e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/filegroup/Filegroup.java @@ -0,0 +1,103 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.filegroup; + +import com.google.devtools.build.lib.actions.Actions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.CompilationHelper; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.MiddlemanProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProviderImpl; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Iterator; + +/** + * ConfiguredTarget for "filegroup". + */ +public class Filegroup implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + NestedSet<Artifact> filesToBuild = NestedSetBuilder.wrap(Order.STABLE_ORDER, + ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list()); + NestedSet<Artifact> middleman = CompilationHelper.getAggregatingMiddleman( + ruleContext, Actions.escapeLabel(ruleContext.getLabel()), filesToBuild); + + InstrumentedFilesCollector instrumentedFilesCollector = + new InstrumentedFilesCollector(ruleContext, + // what do *we* know about whether this is a source file or not + new InstrumentationSpec(FileTypeSet.ANY_FILE, "srcs", "deps", "data"), + InstrumentedFilesCollector.NO_METADATA_COLLECTOR, filesToBuild); + + RunfilesProvider runfilesProvider = RunfilesProvider.withData( + new Runfiles.Builder() + .addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES) + .build(), + // If you're visiting a filegroup as data, then we also visit its data as data. + new Runfiles.Builder().addTransitiveArtifacts(filesToBuild) + .addDataDeps(ruleContext).build()); + + return new RuleConfiguredTargetBuilder(ruleContext) + .add(RunfilesProvider.class, runfilesProvider) + .setFilesToBuild(filesToBuild) + .setRunfilesSupport(null, getExecutable(filesToBuild)) + .add(InstrumentedFilesProvider.class, new InstrumentedFilesProviderImpl( + instrumentedFilesCollector)) + .add(MiddlemanProvider.class, new MiddlemanProvider(middleman)) + .add(FilegroupPathProvider.class, + new FilegroupPathProvider(getFilegroupPath(ruleContext))) + .build(); + } + + /* + * Returns the single executable output of this filegroup. Returns + * {@code null} if there are multiple outputs or the single output is not + * considered an executable. + */ + private Artifact getExecutable(NestedSet<Artifact> filesToBuild) { + Iterator<Artifact> it = filesToBuild.iterator(); + if (it.hasNext()) { + Artifact out = it.next(); + if (!it.hasNext()) { + return out; + } + } + return null; + } + + private PathFragment getFilegroupPath(RuleContext ruleContext) { + String attr = ruleContext.attributes().get("path", Type.STRING); + if (attr.isEmpty()) { + return PathFragment.EMPTY_FRAGMENT; + } else { + return ruleContext.getLabel().getPackageFragment().getRelative(attr); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/filegroup/FilegroupPathProvider.java b/src/main/java/com/google/devtools/build/lib/rules/filegroup/FilegroupPathProvider.java new file mode 100644 index 0000000000..370be07cae --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/filegroup/FilegroupPathProvider.java @@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.filegroup; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * A transitive info provider for dependent targets to query {@code path} attributes. + */ +@Immutable +public final class FilegroupPathProvider implements TransitiveInfoProvider { + private final PathFragment pathFragment; + + public FilegroupPathProvider(PathFragment pathFragment) { + this.pathFragment = pathFragment; + } + + /** + * Returns the value of the {@code path} attribute or the empty fragment if it is not present. + */ + public PathFragment getFilegroupPath() { + return pathFragment; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContext.java new file mode 100644 index 0000000000..056b61e4d4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContext.java @@ -0,0 +1,34 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.fileset; + +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +import java.util.concurrent.ThreadPoolExecutor; + +/** + * Action context for fileset collection actions. + */ +public interface FilesetActionContext extends ActionContext { + + /** + * Returns a thread pool for fileset symlink tree creation. + */ + ThreadPoolExecutor getFilesetPool(); + + /** + * Returns the name of the workspace the build is run in. + */ + String getWorkspaceName(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContextImpl.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContextImpl.java new file mode 100644 index 0000000000..9c03129758 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContextImpl.java @@ -0,0 +1,101 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.fileset; + +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.devtools.build.lib.actions.ActionContextProvider; +import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BlazeExecutor; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.events.Reporter; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Context for Fileset manifest actions. It currently only provides a ThreadPoolExecutor. + * + * <p>Fileset is a legacy, google-internal mechanism to make parts of the source tree appear as a + * tree in the output directory. + */ +@ExecutionStrategy(contextType = FilesetActionContext.class) +public final class FilesetActionContextImpl implements FilesetActionContext { + // TODO(bazel-team): it would be nice if this weren't shipped in Bazel at all. + + /** + * Factory class. + */ + public static class Provider implements ActionContextProvider { + private FilesetActionContextImpl impl; + private final Reporter reporter; + private final ThreadPoolExecutor filesetPool; + + public Provider(Reporter reporter, String workspaceName) { + this.reporter = reporter; + this.filesetPool = newFilesetPool(100); + this.impl = new FilesetActionContextImpl(filesetPool, workspaceName); + } + + private static ThreadPoolExecutor newFilesetPool(int threads) { + ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 3L, TimeUnit.SECONDS, + new LinkedBlockingQueue<Runnable>()); + // Do not consume threads when not in use. + pool.allowCoreThreadTimeOut(true); + pool.setThreadFactory(new ThreadFactoryBuilder().setNameFormat("Fileset worker %d").build()); + return pool; + } + + @Override + public Iterable<ActionContext> getActionContexts() { + return ImmutableList.<ActionContext>of(impl); + } + + @Override + public void executorCreated(Iterable<ActionContext> usedStrategies) {} + + @Override + public void executionPhaseStarting( + ActionInputFileCache actionInputFileCache, + ActionGraph actionGraph, + Iterable<Artifact> topLevelArtifacts) {} + + @Override + public void executionPhaseEnding() { + BlazeExecutor.shutdownHelperPool(reporter, filesetPool, "Fileset"); + } + } + + private final ThreadPoolExecutor filesetPool; + private final String workspaceName; + + private FilesetActionContextImpl(ThreadPoolExecutor filesetPool, String workspaceName) { + this.filesetPool = filesetPool; + this.workspaceName = workspaceName; + } + + @Override + public ThreadPoolExecutor getFilesetPool() { + return filesetPool; + } + + @Override + public String getWorkspaceName() { + return workspaceName; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetLinks.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetLinks.java new file mode 100644 index 0000000000..d523edc356 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetLinks.java @@ -0,0 +1,218 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.fileset; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.syntax.FilesetEntry; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * FilesetLinks manages the set of links added to a Fileset. If two links conflict, the first wins. + * + * <p>FilesetLinks is FileSystem-aware. For example, if you first create a link + * (a/b/c, foo), a subsequent call to link (a/b, bar) is a no-op. + * This is because the first link requires us to create a directory "a/b", + * so "a/b" cannot also link to "bar". + * + * <p>TODO(bazel-team): Consider warning if we have such a conflict; we don't do that currently. + */ +public interface FilesetLinks { + + /** + * Get late directory information for a source. + * + * @param src The source to search for. + * @return The late directory info, or null if none was found. + */ + public LateDirectoryInfo getLateDirectoryInfo(PathFragment src); + + public boolean putLateDirectoryInfo(PathFragment src, LateDirectoryInfo lateDir); + + /** + * Add specified file as a symlink. + * + * The behavior when the target file is a symlink depends on the + * symlinkBehavior parameter (see comments for FilesetEntry.SymlinkBehavior). + * + * @param src The root-relative symlink path. + * @param target The symlink target. + */ + public void addFile(PathFragment src, Path target, String metadata, + FilesetEntry.SymlinkBehavior symlinkBehavior) + throws IOException; + + /** + * Add all late directories as symlinks. This function should be called only + * after all recursions have completed, but before getData or getSymlinks are + * called. + */ + public void addLateDirectories() throws IOException; + + /** + * Adds the given symlink to the tree. + * + * @param fromFrag The root-relative symlink path. + * @param toFrag The symlink target. + * @return true iff the symlink was added. + */ + public boolean addLink(PathFragment fromFrag, PathFragment toFrag, String dataVal); + + /** + * @return The unmodifiable map of symlinks. + */ + public Map<PathFragment, PathFragment> getSymlinks(); + + /** + * @return The unmodifiable map of metadata. + */ + public Map<PathFragment, String> getData(); + + /** + * A data structure for containing all the information about a directory that + * is late-added. This means the directory is skipped unless we need to + * recurse into it later. If the directory is never recursed into, we will + * create a symlink directly to it. + */ + public static final class LateDirectoryInfo { + // The constructors are private. Use the factory functions below to create + // instances of this class. + + /** Construct a stub LateDirectoryInfo object. */ + private LateDirectoryInfo() { + this.added = new AtomicBoolean(true); + + // Shut up the compiler. + this.target = null; + this.src = null; + this.pkgMode = SubpackageMode.IGNORE; + this.metadata = null; + this.symlinkBehavior = null; + } + + /** Construct a normal LateDirectoryInfo object. */ + private LateDirectoryInfo(Path target, PathFragment src, SubpackageMode pkgMode, + String metadata, FilesetEntry.SymlinkBehavior symlinkBehavior) { + this.target = target; + this.src = src; + this.pkgMode = pkgMode; + this.metadata = metadata; + this.symlinkBehavior = symlinkBehavior; + this.added = new AtomicBoolean(false); + } + + /** @return The target path for the symlink. The target is the referent. */ + public Path getTarget() { + return target; + } + + /** + * @return The source path for the symlink. The source is the place the + * symlink will be written. */ + public PathFragment getSrc() { + return src; + } + + /** + * @return Whether we should show a warning if we cross a package boundary + * when recursing into this directory. + */ + public SubpackageMode getPkgMode() { + return pkgMode; + } + + /** + * @return The metadata we will write into the manifest if we symlink to + * this directory. + */ + public String getMetadata() { + return metadata; + } + + /** + * @return How to perform the symlinking if the source happens to be a + * symlink itself. + */ + public FilesetEntry.SymlinkBehavior getTargetSymlinkBehavior() { + return Preconditions.checkNotNull(symlinkBehavior, + "should not call this method on stub instances"); + } + + /** + * Atomically checks if the late directory has been added to the manifest + * and marks it as added. If this function returns true, it is the + * responsibility of the caller to recurse into the late directory. + * Otherwise, some other caller has already, or is in the process of + * recursing into it. + * @return Whether the caller should recurse into the late directory. + */ + public boolean shouldAdd() { + return !added.getAndSet(true); + } + + /** + * Create a stub LateDirectoryInfo that is already marked as added. + * @return The new LateDirectoryInfo object. + */ + public static LateDirectoryInfo createStub() { + return new LateDirectoryInfo(); + } + + /** + * Create a LateDirectoryInfo object with the specified attributes. + * @param target The directory to which the symlinks will refer. + * @param src The location at which to create the symlink. + * @param pkgMode How to handle recursion into another package. + * @param metadata The metadata for the directory to write into the + * manifest if we symlink it directly. + * @return The new LateDirectoryInfo object. + */ + public static LateDirectoryInfo create(Path target, PathFragment src, SubpackageMode pkgMode, + String metadata, FilesetEntry.SymlinkBehavior symlinkBehavior) { + return new LateDirectoryInfo(target, src, pkgMode, metadata, symlinkBehavior); + } + + /** + * The target directory to which the symlink will point. + * Note this is a real path on the filesystem and can't be compared to src + * or any source (key) in the links map. + */ + private final Path target; + + /** The referent of the symlink. */ + private final PathFragment src; + + /** Whether to show cross package boundary warnings / errors. */ + private final SubpackageMode pkgMode; + + /** The metadata to write into the manifest file. */ + private final String metadata; + + /** How to perform the symlinking if the source happens to be a symlink itself. */ + private final FilesetEntry.SymlinkBehavior symlinkBehavior; + + /** Whether the directory has already been recursed into. */ + private final AtomicBoolean added; + } + + /** How to handle filesets that cross subpackages. */ + public static enum SubpackageMode { + ERROR, WARNING, IGNORE; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetProvider.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetProvider.java new file mode 100644 index 0000000000..6b70aab267 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetProvider.java @@ -0,0 +1,27 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.fileset; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Information needed by a Fileset to do the right thing when it depends on another Fileset. + */ +public interface FilesetProvider extends TransitiveInfoProvider { + Artifact getFilesetInputManifest(); + PathFragment getFilesetLinkDir(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/SymlinkTraversal.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/SymlinkTraversal.java new file mode 100644 index 0000000000..db13bdb3c4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/SymlinkTraversal.java @@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.fileset; + +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; + +import java.io.IOException; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * An interface which contains a method to compute a symlink mapping. + */ +public interface SymlinkTraversal { + + /** + * Adds symlinks to the given FilesetLinks. + * + * @throws IOException if a filesystem operation fails. + * @throws InterruptedException if the traversal is interrupted. + */ + void addSymlinks(EventHandler eventHandler, FilesetLinks links, ThreadPoolExecutor filesetPool) + throws IOException, InterruptedException; + + /** + * Add the traversal's fingerprint to the given Fingerprint. + * @param fp the Fingerprint to combine. + */ + void fingerprint(Fingerprint fp); + + /** + * @return true iff this traversal must be executed unconditionally. + */ + boolean executeUnconditionally(); + + /** + * Returns true if it's ever possible that {@link #executeUnconditionally} + * could evaluate to true during the lifetime of this instance, false + * otherwise. + */ + boolean isVolatile(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/BaseJavaCompilationHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/BaseJavaCompilationHelper.java new file mode 100644 index 0000000000..69dc41b5c9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/BaseJavaCompilationHelper.java @@ -0,0 +1,237 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; + +/** + * A helper class for compiling Java targets. This helper does not rely on the + * presence of rule-specific attributes. + */ +public class BaseJavaCompilationHelper { + /** + * Also see DeployArchiveBuilder.SINGLEJAR_MAX_MEMORY. We don't expect that anyone has more + * than ~500,000 files in a source jar, so 256 MB of memory should be plenty. + */ + private static final String SINGLEJAR_MAX_MEMORY = "-Xmx256m"; + + private final RuleContext ruleContext; + + public BaseJavaCompilationHelper(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + /** + * Returns the artifacts required to invoke {@code javahome} relative binary + * in the action. + */ + public static NestedSet<Artifact> getHostJavabaseInputs(RuleContext ruleContext) { + // This must have a different name than above, because the middleman creation uses the rule's + // configuration, although it should use the host configuration. + return AnalysisUtils.getMiddlemanFor(ruleContext, ":host_jdk"); + } + + private static final ImmutableList<String> SOURCE_JAR_COMMAND_LINE_ARGS = ImmutableList.of( + "--compression", + "--normalize", + "--exclude_build_data", + "--warn_duplicate_resources"); + + private CommandLine sourceJarCommandLine(JavaSemantics semantics, Artifact outputJar, + Iterable<Artifact> resources, Iterable<Artifact> resourceJars) { + CustomCommandLine.Builder args = CustomCommandLine.builder(); + args.addExecPath("--output", outputJar); + args.add(SOURCE_JAR_COMMAND_LINE_ARGS); + args.addExecPaths("--sources", resourceJars); + args.add("--resources"); + for (Artifact resource : resources) { + args.addPaths("%s:%s", resource.getExecPath(), + semantics.getJavaResourcePath(resource.getRootRelativePath())); + } + return args.build(); + } + + /** + * Creates an Action that packages files into a Jar file. + * + * @param semantics delegate semantics for java. + * @param resources the resources to put into the Jar. + * @param resourceJars the resource jars to merge into the jar + * @param outputJar the Jar to create + */ + public void createSourceJarAction(JavaSemantics semantics, Collection<Artifact> resources, + Collection<Artifact> resourceJars, Artifact outputJar) { + ruleContext.registerAction(new SpawnAction.Builder() + .addOutput(outputJar) + .addInputs(resources) + .addInputs(resourceJars) + .addTransitiveInputs(JavaCompilationHelper.getHostJavabaseInputs(ruleContext)) + .setJarExecutable( + ruleContext.getHostConfiguration().getFragment(Jvm.class).getJavaExecutable(), + ruleContext.getPrerequisiteArtifact("$singlejar", Mode.HOST), + ImmutableList.of("-client", SINGLEJAR_MAX_MEMORY)) + .setCommandLine(sourceJarCommandLine(semantics, outputJar, resources, resourceJars)) + .useParameterFile(ParameterFileType.SHELL_QUOTED) + .setProgressMessage("Building source jar " + outputJar.prettyPrint()) + .setMnemonic("JavaSourceJar") + .build(ruleContext)); + } + + /** + * Returns the langtools jar Artifact. + */ + protected final Artifact getLangtoolsJar() { + return ruleContext.getHostPrerequisiteArtifact("$java_langtools"); + } + + /** + * Returns the JavaBuilder jar Artifact. + */ + protected final Artifact getJavaBuilderJar() { + return ruleContext.getPrerequisiteArtifact("$javabuilder", Mode.HOST); + } + + /** + * Returns the javac bootclasspath artifacts. + */ + protected final Iterable<Artifact> getBootClasspath() { + return ruleContext.getPrerequisiteArtifacts("$javac_bootclasspath", Mode.HOST).list(); + } + + private Artifact getIjarArtifact(Artifact jar, boolean addPrefix) { + if (addPrefix) { + PathFragment ruleBase = ruleContext.getLabel().getPackageFragment().getRelative( + ruleContext.getLabel().getName()).getRelative("_ijars"); + PathFragment artifactDirFragment = jar.getRootRelativePath().getParentDirectory(); + String ijarBasename = FileSystemUtils.removeExtension(jar.getFilename()) + "-ijar.jar"; + return getAnalysisEnvironment().getDerivedArtifact( + ruleBase.getRelative(artifactDirFragment).getRelative(ijarBasename), + getConfiguration().getGenfilesDirectory()); + } else { + return derivedArtifact(jar, "", "-ijar.jar"); + } + } + + /** + * Creates the Action that creates ijars from Jar files. + * + * @param inputJar the Jar to create the ijar for + * @param addPrefix whether to prefix the path of the generated ijar with the package and + * name of the current rule + * @return the Artifact to create with the Action + */ + protected Artifact createIjarAction(final Artifact inputJar, boolean addPrefix) { + Artifact interfaceJar = getIjarArtifact(inputJar, addPrefix); + final FilesToRunProvider ijarTarget = + ruleContext.getExecutablePrerequisite("$ijar", Mode.HOST); + if (!ruleContext.hasErrors()) { + ruleContext.registerAction(new SpawnAction.Builder() + .addInput(inputJar) + .addOutput(interfaceJar) + .setExecutable(ijarTarget) + .addArgument(inputJar.getExecPathString()) + .addArgument(interfaceJar.getExecPathString()) + .setProgressMessage("Extracting interface " + ruleContext.getLabel()) + .setMnemonic("JavaIjar") + .build(ruleContext)); + } + return interfaceJar; + } + + protected final JavaCompileAction.Builder createJavaCompileActionBuilder( + JavaSemantics semantics) { + JavaCompileAction.Builder builder = new JavaCompileAction.Builder(ruleContext, semantics); + builder.setJavaExecutable( + ruleContext.getHostConfiguration().getFragment(Jvm.class).getJavaExecutable()); + builder.setJavaBaseInputs(BaseJavaCompilationHelper.getHostJavabaseInputs(ruleContext)); + return builder; + } + + public RuleContext getRuleContext() { + return ruleContext; + } + + public AnalysisEnvironment getAnalysisEnvironment() { + return ruleContext.getAnalysisEnvironment(); + } + + protected BuildConfiguration getConfiguration() { + return ruleContext.getConfiguration(); + } + + protected JavaConfiguration getJavaConfiguration() { + return ruleContext.getFragment(JavaConfiguration.class); + } + + protected PathFragment outputDir(Artifact outputJar) { + return workDir(outputJar, "_files"); + } + + /** + * Produces a derived directory where source files generated by annotation processors should be + * stored. + */ + protected PathFragment sourceGenDir(Artifact outputJar) { + return workDir(outputJar, "_sourcegenfiles"); + } + + protected PathFragment tempDir(Artifact outputJar) { + return workDir(outputJar, "_temp"); + } + + /** + * For an output jar and a suffix, produces a derived directory under + * {@code bin} directory with a given suffix. + */ + private PathFragment workDir(Artifact outputJar, String suffix) { + PathFragment path = outputJar.getRootRelativePath(); + String basename = FileSystemUtils.removeExtension(path.getBaseName()) + suffix; + path = path.replaceName(basename); + return getConfiguration().getBinDirectory().getExecPath().getRelative(path); + } + + /** + * Creates a derived artifact from the given artifact by adding the given + * prefix and removing the extension and replacing it by the given suffix. + * The new artifact will have the same root as the given one. + */ + protected Artifact derivedArtifact(Artifact artifact, String prefix, String suffix) { + return derivedArtifact(artifact, prefix, suffix, artifact.getRoot()); + } + + protected Artifact derivedArtifact(Artifact artifact, String prefix, String suffix, Root root) { + PathFragment path = artifact.getRootRelativePath(); + String basename = FileSystemUtils.removeExtension(path.getBaseName()) + suffix; + path = path.replaceName(prefix + basename); + return getAnalysisEnvironment().getDerivedArtifact(path, root); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/BuildInfoPropertiesTranslator.java b/src/main/java/com/google/devtools/build/lib/rules/java/BuildInfoPropertiesTranslator.java new file mode 100644 index 0000000000..053b9e76bc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/BuildInfoPropertiesTranslator.java @@ -0,0 +1,33 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import java.util.Map; +import java.util.Properties; + +/** + * A class to describe how build information should be translated into the generated properties + * file. + */ +public interface BuildInfoPropertiesTranslator { + + /** Translate build information into a property file. */ + public void translate(Map<String, String> buildInfo, Properties properties); + + /** + * Returns a unique key for this translator to be used by the + * {@link com.google.devtools.build.lib.actions.Action#getKey()} method + */ + public String computeKey(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/ClasspathConfiguredFragment.java b/src/main/java/com/google/devtools/build/lib/rules/java/ClasspathConfiguredFragment.java new file mode 100644 index 0000000000..6510a491c9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/ClasspathConfiguredFragment.java @@ -0,0 +1,96 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; + +/** + * Represents common aspects of all JVM targeting configured targets. + */ +public final class ClasspathConfiguredFragment { + + private final NestedSet<Artifact> runtimeClasspath; + private final NestedSet<Artifact> compileTimeClasspath; + private final ImmutableList<Artifact> bootClasspath; + + /** + * Initializes the runtime and compile time classpaths for this target. This method + * should be called during {@code initializationHook()} once a {@link JavaTargetAttributes} + * object for this target is fully initialized. + * + * @param attributes the processed attributes of this Java target + * @param isNeverLink whether to leave runtimeClasspath empty + */ + public ClasspathConfiguredFragment(JavaCompilationArtifacts javaArtifacts, + JavaTargetAttributes attributes, boolean isNeverLink) { + if (!isNeverLink) { + runtimeClasspath = getRuntimeClasspathList(attributes, javaArtifacts); + } else { + runtimeClasspath = NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER); + } + compileTimeClasspath = attributes.getCompileTimeClassPath(); + bootClasspath = attributes.getBootClassPath(); + } + + public ClasspathConfiguredFragment() { + runtimeClasspath = NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER); + compileTimeClasspath = NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER); + bootClasspath = ImmutableList.of(); + } + + /** + * Returns the runtime class path. It consists of the concatenation of the + * instrumentation class path, output jars and the runtime time class path of + * the transitive dependencies of this rule. + * + * @param attributes the processed attributes of this Java target + * + * @return a {@List} of artifacts that comprise the runtime class path. + */ + private NestedSet<Artifact> getRuntimeClasspathList( + JavaTargetAttributes attributes, JavaCompilationArtifacts javaArtifacts) { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.naiveLinkOrder(); + builder.addAll(javaArtifacts.getRuntimeJars()); + builder.addTransitive(attributes.getRuntimeClassPath()); + return builder.build(); + } + + /** + * Returns the classpath to be passed to the JVM when running a target containing this fragment. + */ + public NestedSet<Artifact> getRuntimeClasspath() { + return runtimeClasspath; + } + + /** + * Returns the classpath to be passed to the Java compiler when compiling a target containing this + * fragment. + */ + public NestedSet<Artifact> getCompileTimeClasspath() { + return compileTimeClasspath; + } + + /** + * Returns the classpath to be passed as a boot classpath to the Java compiler when compiling + * a target containing this fragment. + */ + public ImmutableList<Artifact> getBootClasspath() { + return bootClasspath; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/DeployArchiveBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/java/DeployArchiveBuilder.java new file mode 100644 index 0000000000..b9fe186ebd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/DeployArchiveBuilder.java @@ -0,0 +1,256 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.collect.IterablesChain; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Utility for configuring an action to generate a deploy archive. + */ +public class DeployArchiveBuilder { + /** + * Memory consumption of SingleJar is about 250 bytes per entry in the output file. Unfortunately, + * the JVM tends to kill the process with an OOM long before we're at the limit. In the most + * recent example, 400 MB of memory was enough for about 500,000 entries. + */ + private static final String SINGLEJAR_MAX_MEMORY = "-Xmx1600m"; + + private final RuleContext ruleContext; + + private final IterablesChain.Builder<Artifact> runtimeJarsBuilder = IterablesChain.builder(); + + private final JavaSemantics semantics; + + private JavaTargetAttributes attributes; + private boolean includeBuildData; + private Compression compression = Compression.UNCOMPRESSED; + @Nullable private Artifact runfilesMiddleman; + private Artifact outputJar; + @Nullable private String javaStartClass; + private ImmutableList<String> deployManifestLines = ImmutableList.of(); + @Nullable private Artifact launcher; + + /** + * Type of compression to apply to output archive. + */ + public enum Compression { + + /** Output should be compressed */ + COMPRESSED, + + /** Output should not be compressed */ + UNCOMPRESSED; + } + + /** + * Creates a builder using the configuration of the rule as the action configuration. + */ + public DeployArchiveBuilder(JavaSemantics semantics, RuleContext ruleContext) { + this.ruleContext = ruleContext; + this.semantics = semantics; + } + + /** + * Sets the processed attributes of the rule generating the deploy archive. + */ + public DeployArchiveBuilder setAttributes(JavaTargetAttributes attributes) { + this.attributes = attributes; + return this; + } + + /** + * Sets whether to include build-data.properties in the deploy archive. + */ + public DeployArchiveBuilder setIncludeBuildData(boolean includeBuildData) { + this.includeBuildData = includeBuildData; + return this; + } + + /** + * Sets whether to enable compression of the output deploy archive. + */ + public DeployArchiveBuilder setCompression(Compression compress) { + this.compression = Preconditions.checkNotNull(compress); + return this; + } + + /** + * Sets additional dependencies to be added to the action that creates the + * deploy jar so that we force the runtime dependencies to be built. + */ + public DeployArchiveBuilder setRunfilesMiddleman(@Nullable Artifact runfilesMiddleman) { + this.runfilesMiddleman = runfilesMiddleman; + return this; + } + + /** + * Sets the artifact to create with the action. + */ + public DeployArchiveBuilder setOutputJar(Artifact outputJar) { + this.outputJar = Preconditions.checkNotNull(outputJar); + return this; + } + + /** + * Sets the class to launch the Java application. + */ + public DeployArchiveBuilder setJavaStartClass(@Nullable String javaStartClass) { + this.javaStartClass = javaStartClass; + return this; + } + + /** + * Adds additional jars that should be on the classpath at runtime. + */ + public DeployArchiveBuilder addRuntimeJars(Iterable<Artifact> jars) { + this.runtimeJarsBuilder.add(jars); + return this; + } + + /** + * Sets the list of extra lines to add to the archive's MANIFEST.MF file. + */ + public DeployArchiveBuilder setDeployManifestLines(Iterable<String> deployManifestLines) { + this.deployManifestLines = ImmutableList.copyOf(deployManifestLines); + return this; + } + + /** + * Sets the optional launcher to be used as the executable for this deploy + * JAR + */ + public DeployArchiveBuilder setLauncher(@Nullable Artifact launcher) { + this.launcher = launcher; + return this; + } + + public static CustomCommandLine.Builder defaultSingleJarCommandLine(Artifact outputJar, + String javaMainClass, + ImmutableList<String> deployManifestLines, Iterable<Artifact> buildInfoFiles, + ImmutableList<Artifact> classpathResources, + Iterable<Artifact> runtimeClasspath, boolean includeBuildData, + Compression compress, Artifact launcher) { + + CustomCommandLine.Builder args = CustomCommandLine.builder(); + args.addExecPath("--output", outputJar); + if (compress == Compression.COMPRESSED) { + args.add("--compression"); + } + args.add("--normalize"); + if (javaMainClass != null) { + args.add("--main_class"); + args.add(javaMainClass); + } + + if (!deployManifestLines.isEmpty()) { + args.add("--deploy_manifest_lines"); + args.add(deployManifestLines); + } + + if (buildInfoFiles != null) { + for (Artifact artifact : buildInfoFiles) { + args.addExecPath("--build_info_file", artifact); + } + } + if (!includeBuildData) { + args.add("--exclude_build_data"); + } + if (launcher != null) { + args.add("--java_launcher"); + args.add(launcher.getExecPathString()); + } + + args.addExecPaths("--classpath_resources", classpathResources); + args.addExecPaths("--sources", runtimeClasspath); + return args; + } + + /** + * Builds the action as configured. + */ + public void build() { + ImmutableList<Artifact> classpathResources = attributes.getClassPathResources(); + Set<String> classPathResourceNames = new HashSet<>(); + for (Artifact artifact : classpathResources) { + String name = artifact.getExecPath().getBaseName(); + if (!classPathResourceNames.add(name)) { + ruleContext.attributeError("classpath_resources", + "entries must have different file names (duplicate: " + name + ")"); + return; + } + } + + IterablesChain<Artifact> runtimeJars = runtimeJarsBuilder.build(); + + IterablesChain.Builder<Artifact> inputs = IterablesChain.builder(); + inputs.add(attributes.getArchiveInputs(true)); + + inputs.add(ImmutableList.copyOf(runtimeJars)); + if (runfilesMiddleman != null) { + inputs.addElement(runfilesMiddleman); + } + + final ImmutableList<Artifact> buildInfoArtifacts = + ruleContext.getAnalysisEnvironment().getBuildInfo(ruleContext, JavaBuildInfoFactory.KEY); + inputs.add(buildInfoArtifacts); + + Iterable<Artifact> runtimeClasspath = Iterables.concat( + runtimeJars, + attributes.getRuntimeClassPathForArchive()); + + if (launcher != null) { + inputs.addElement(launcher); + } + + CommandLine commandLine = semantics.buildSingleJarCommandLine(ruleContext.getConfiguration(), + outputJar, javaStartClass, deployManifestLines, buildInfoArtifacts, classpathResources, + runtimeClasspath, includeBuildData, compression, launcher); + + List<String> jvmArgs = ImmutableList.of("-client", SINGLEJAR_MAX_MEMORY); + ResourceSet resourceSet = + new ResourceSet(/*memoryMb = */200.0, /*cpuUsage = */.2, /*ioUsage=*/.2); + + ruleContext.registerAction(new SpawnAction.Builder() + .addInputs(inputs.build()) + .addTransitiveInputs(JavaCompilationHelper.getHostJavabaseInputs(ruleContext)) + .addOutput(outputJar) + .setResources(resourceSet) + .setJarExecutable( + ruleContext.getHostConfiguration().getFragment(Jvm.class).getJavaExecutable(), + ruleContext.getPrerequisiteArtifact("$singlejar", Mode.HOST), + jvmArgs) + .setCommandLine(commandLine) + .useParameterFile(ParameterFileType.SHELL_QUOTED) + .setProgressMessage("Building deploy jar " + outputJar.prettyPrint()) + .setMnemonic("JavaDeployJar") + .build(ruleContext)); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/DirectDependencyProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/DirectDependencyProvider.java new file mode 100644 index 0000000000..5b2e106c8e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/DirectDependencyProvider.java @@ -0,0 +1,64 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; + +/** + * A provider that returns the direct dependencies of a target. Used for strict dependency + * checking. + */ +@Immutable +public final class DirectDependencyProvider implements TransitiveInfoProvider { + + private final ImmutableList<Dependency> strictDependencies; + + public DirectDependencyProvider(Iterable<Dependency> strictDependencies) { + this.strictDependencies = ImmutableList.copyOf(strictDependencies); + } + + /** + * @returns the direct (strict) dependencies of this provider. All symbols that are directly + * reachable from the sources of the provider should be available in one these artifacts. + */ + public Iterable<Dependency> getStrictDependencies() { + return strictDependencies; + } + + /** + * A pair of label and its generated list of artifacts. + */ + public static class Dependency { + private final Label label; + + // TODO(bazel-team): change this to Artifacts + private final Iterable<String> fileExecPaths; + + public Dependency(Label label, Iterable<String> fileExecPaths) { + this.label = label; + this.fileExecPaths = fileExecPaths; + } + + public Label getLabel() { + return label; + } + + public Iterable<String> getDependencyOutputs() { + return fileExecPaths; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/GenericBuildInfoPropertiesTranslator.java b/src/main/java/com/google/devtools/build/lib/rules/java/GenericBuildInfoPropertiesTranslator.java new file mode 100644 index 0000000000..df6a325437 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/GenericBuildInfoPropertiesTranslator.java @@ -0,0 +1,91 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.util.Fingerprint; + +import java.util.Map; +import java.util.Properties; + +/** The generic implementation of {@link BuildInfoPropertiesTranslator} */ +public class GenericBuildInfoPropertiesTranslator implements + BuildInfoPropertiesTranslator { + + private static final String GUID = "e71fe4a8-11af-4ec0-9b38-1d3e7f542f51"; + + // syntax is %ID% for a property that depends on the ID key, %ID|default% to + // always add the property with the "default" key, %% is to add a percent sign + private final Map<String, String> translationKeys; + + /** + * Create a generic translator, for each key,value pair in {@code translationKeys}, the key + * represents the property key that will be written and the value, its value. Inside value every + * %ID% is replaced by the corresponding build information with the same ID key. The property + * won't be added if it's depends on an unresolved build information. Adding a property can + * be forced even if a build information is missing by specifying a default value using the + * %ID|default% syntax. Finally to add a percent sign, just use the %% syntax. + */ + public GenericBuildInfoPropertiesTranslator(Map<String, String> translationKeys) { + this.translationKeys = translationKeys; + } + + @Override + public String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addStringMap(translationKeys); + return f.hexDigestAndReset(); + } + + @Override + public void translate(Map<String, String> buildInfo, Properties properties) { + for (Map.Entry<String, String> entry : translationKeys.entrySet()) { + String translatedValue = translateValue(entry.getValue(), buildInfo); + if (translatedValue != null) { + properties.put(entry.getKey(), translatedValue); + } + } + } + + private String translateValue(String valueDescription, Map<String, String> buildInfo) { + String[] split = valueDescription.split("%"); + StringBuffer result = new StringBuffer(); + boolean isInsideKey = false; + for (String key : split) { + if (isInsideKey) { + if (key.isEmpty()) { + result.append("%"); // empty key means %% + } else { + String defaultValue = null; + int i = key.lastIndexOf('|'); + if (i >= 0) { + defaultValue = key.substring(i + 1); + key = key.substring(0, i); + } + if (buildInfo.containsKey(key)) { + result.append(buildInfo.get(key)); + } else if (defaultValue != null) { + result.append(defaultValue); + } else { // we haven't found the requested key so we ignore the whole value + return null; + } + } + } else { + result.append(key); + } + isInsideKey = !isInsideKey; + } + return result.toString(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java new file mode 100644 index 0000000000..e957f49d84 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java @@ -0,0 +1,359 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import static com.google.devtools.build.lib.rules.java.DeployArchiveBuilder.Compression.COMPRESSED; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.TopLevelArtifactProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.cpp.CppHelper; +import com.google.devtools.build.lib.rules.cpp.LinkerInput; +import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * An implementation of java_binary. + */ +public class JavaBinary implements RuleConfiguredTargetFactory { + private static final PathFragment CPP_RUNTIMES = new PathFragment("_cpp_runtimes"); + + private final JavaSemantics semantics; + + protected JavaBinary(JavaSemantics semantics) { + this.semantics = semantics; + } + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + final JavaCommon common = new JavaCommon(ruleContext, semantics); + DeployArchiveBuilder deployArchiveBuilder = new DeployArchiveBuilder(semantics, ruleContext); + Runfiles.Builder runfilesBuilder = new Runfiles.Builder(); + List<String> jvmFlags = new ArrayList<>(); + + common.initializeJavacOpts(); + JavaTargetAttributes.Builder attributesBuilder = common.initCommon(); + attributesBuilder.addClassPathResources( + ruleContext.getPrerequisiteArtifacts("classpath_resources", Mode.TARGET).list()); + + List<String> userJvmFlags = common.getJvmFlags(); + + ruleContext.checkSrcsSamePackage(true); + boolean createExecutable = ruleContext.attributes().get("create_executable", Type.BOOLEAN); + List<TransitiveInfoCollection> deps = + Lists.newArrayList(common.targetsTreatedAsDeps(ClasspathType.COMPILE_ONLY)); + semantics.checkRule(ruleContext, common); + String mainClass = semantics.getMainClass(ruleContext, common); + String originalMainClass = mainClass; + if (ruleContext.hasErrors()) { + return null; + } + + // Collect the transitive dependencies. + JavaCompilationHelper helper = new JavaCompilationHelper( + ruleContext, semantics, common.getJavacOpts(), attributesBuilder); + helper.addLibrariesToAttributes(deps); + helper.addProvidersToAttributes(common.compilationArgsFromSources(), /* isNeverLink */ false); + attributesBuilder.addNativeLibraries( + collectNativeLibraries(common.targetsTreatedAsDeps(ClasspathType.BOTH))); + + // deploy_env is valid for java_binary, but not for java_test. + if (ruleContext.getRule().isAttrDefined("deploy_env", Type.LABEL_LIST)) { + for (JavaRuntimeClasspathProvider envTarget : ruleContext.getPrerequisites( + "deploy_env", Mode.TARGET, JavaRuntimeClasspathProvider.class)) { + attributesBuilder.addExcludedArtifacts(envTarget.getRuntimeClasspath()); + } + } + + Artifact srcJar = + ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_SOURCE_JAR); + + Artifact classJar = + ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_CLASS_JAR); + + ImmutableList<Artifact> srcJars = ImmutableList.of(srcJar); + + Artifact launcher = semantics.getLauncher(ruleContext, common, deployArchiveBuilder, + runfilesBuilder, jvmFlags, attributesBuilder); + JavaCompilationArtifacts.Builder javaArtifactsBuilder = new JavaCompilationArtifacts.Builder(); + Artifact instrumentationMetadata = + helper.createInstrumentationMetadata(classJar, javaArtifactsBuilder); + + NestedSetBuilder<Artifact> filesBuilder = NestedSetBuilder.stableOrder(); + Artifact executable = null; + if (createExecutable) { + executable = ruleContext.createOutputArtifact(); // the artifact for the rule itself + filesBuilder.add(classJar).add(executable); + + if (ruleContext.getConfiguration().isCodeCoverageEnabled()) { + mainClass = semantics.addCoverageSupport(helper, attributesBuilder, + executable, instrumentationMetadata, javaArtifactsBuilder, mainClass); + } + } else { + filesBuilder.add(classJar); + } + + JavaTargetAttributes attributes = helper.getAttributes(); + List<Artifact> nativeLibraries = attributes.getNativeLibraries(); + if (!nativeLibraries.isEmpty()) { + jvmFlags.add("-Djava.library.path=" + JavaCommon.javaLibraryPath(nativeLibraries)); + } + + JavaConfiguration javaConfig = ruleContext.getFragment(JavaConfiguration.class); + if (attributes.hasMessages()) { + helper.addTranslations(semantics.translate(ruleContext, javaConfig, + attributes.getMessages())); + } + + if (attributes.hasSourceFiles() || attributes.hasSourceJars() + || attributes.hasResources() || attributes.hasClassPathResources()) { + // We only want to add a jar to the classpath of a dependent rule if it has content. + javaArtifactsBuilder.addRuntimeJar(classJar); + } + + // Any JAR files should be added to the collection of runtime jars. + javaArtifactsBuilder.addRuntimeJars(attributes.getJarFiles()); + + Artifact outputDepsProto = helper.createOutputDepsProtoArtifact(classJar, javaArtifactsBuilder); + + common.setJavaCompilationArtifacts(javaArtifactsBuilder.build()); + + // The gensrcJar is only created if the target uses annotation processing. Otherwise, + // it is null, and the source jar action will not depend on the compile action. + Artifact gensrcJar = helper.createGensrcJar(classJar); + + helper.createCompileAction(classJar, gensrcJar, outputDepsProto, instrumentationMetadata); + helper.createSourceJarAction(srcJar, gensrcJar); + + common.setClassPathFragment(new ClasspathConfiguredFragment( + common.getJavaCompilationArtifacts(), attributes, false)); + + // Collect the action inputs for the runfiles collector here because we need to access the + // analysis environment, and that may no longer be safe when the runfiles collector runs. + Iterable<Artifact> dynamicRuntimeActionInputs = + CppHelper.getToolchain(ruleContext).getDynamicRuntimeLinkInputs(); + + + Iterables.addAll(jvmFlags, semantics.getJvmFlags(ruleContext, common, launcher, userJvmFlags)); + if (ruleContext.hasErrors()) { + return null; + } + + if (createExecutable) { + // Create a shell stub for a Java application + semantics.createStubAction(ruleContext, common, jvmFlags, executable, mainClass, + common.getJavaBinSubstitution(launcher)); + } + + NestedSet<Artifact> transitiveSourceJars = collectTransitiveSourceJars(common, srcJar); + + // TODO(bazel-team): if (getOptions().sourceJars) then make this a dummy prerequisite for the + // DeployArchiveAction ? Needs a few changes there as we can't pass inputs + helper.createSourceJarAction(semantics, ImmutableList.<Artifact>of(), + transitiveSourceJars.toCollection(), + ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_DEPLOY_SOURCE_JAR)); + + RuleConfiguredTargetBuilder builder = + new RuleConfiguredTargetBuilder(ruleContext); + + semantics.addProviders(ruleContext, common, jvmFlags, classJar, srcJar, gensrcJar, + ImmutableMap.<Artifact, Artifact>of(), helper, filesBuilder, builder); + + NestedSet<Artifact> filesToBuild = filesBuilder.build(); + + collectDefaultRunfiles(runfilesBuilder, ruleContext, common, filesToBuild, launcher, + dynamicRuntimeActionInputs); + Runfiles defaultRunfiles = runfilesBuilder.build(); + + RunfilesSupport runfilesSupport = createExecutable + ? runfilesSupport = RunfilesSupport.withExecutable( + ruleContext, defaultRunfiles, executable, + semantics.getExtraArguments(ruleContext, common)) + : null; + + RunfilesProvider runfilesProvider = RunfilesProvider.withData( + defaultRunfiles, + new Runfiles.Builder().merge(runfilesSupport).build()); + + ImmutableList<String> deployManifestLines = + getDeployManifestLines(ruleContext, originalMainClass); + + Artifact deployJar = + ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_DEPLOY_JAR); + + deployArchiveBuilder + .setOutputJar(deployJar) + .setJavaStartClass(mainClass) + .setDeployManifestLines(deployManifestLines) + .setAttributes(attributes) + .addRuntimeJars(common.getJavaCompilationArtifacts().getRuntimeJars()) + .setIncludeBuildData(true) + .setRunfilesMiddleman( + runfilesSupport == null ? null : runfilesSupport.getRunfilesMiddleman()) + .setCompression(COMPRESSED) + .setLauncher(launcher); + + deployArchiveBuilder.build(); + + common.addTransitiveInfoProviders(builder, filesToBuild, classJar); + + return builder + .setFilesToBuild(filesToBuild) + .add(RunfilesProvider.class, runfilesProvider) + .setRunfilesSupport(runfilesSupport, executable) + .add(JavaRuntimeClasspathProvider.class, + new JavaRuntimeClasspathProvider(common.getRuntimeClasspath())) + .add(JavaSourceJarsProvider.class, + new JavaSourceJarsProvider(transitiveSourceJars, srcJars)) + .add(TopLevelArtifactProvider.class, new TopLevelArtifactProvider( + JavaSemantics.SOURCE_JARS_OUTPUT_GROUP, transitiveSourceJars)) + .build(); + } + + // Create the deploy jar and make it dependent on the runfiles middleman if an executable is + // created. Do not add the deploy jar to files to build, so we will only build it when it gets + // requested. + private ImmutableList<String> getDeployManifestLines(RuleContext ruleContext, + String originalMainClass) { + ImmutableList.Builder<String> builder = ImmutableList.<String>builder() + .addAll(ruleContext.attributes().get("deploy_manifest_lines", Type.STRING_LIST)); + if (ruleContext.getConfiguration().isCodeCoverageEnabled()) { + builder.add("Coverage-Main-Class: " + originalMainClass); + } + return builder.build(); + } + + private void collectDefaultRunfiles(Runfiles.Builder builder, RuleContext ruleContext, + JavaCommon common, NestedSet<Artifact> filesToBuild, Artifact launcher, + Iterable<Artifact> dynamicRuntimeActionInputs) { + // Convert to iterable: filesToBuild has a different order. + builder.addArtifacts((Iterable<Artifact>) filesToBuild); + builder.addArtifacts(common.getJavaCompilationArtifacts().getRuntimeJars()); + if (launcher != null) { + final TransitiveInfoCollection defaultLauncher = + JavaHelper.launcherForTarget(semantics, ruleContext); + final Artifact defaultLauncherArtifact = + JavaHelper.launcherArtifactForTarget(semantics, ruleContext); + if (!defaultLauncherArtifact.equals(launcher)) { + builder.addArtifact(launcher); + + // N.B. The "default launcher" referred to here is the launcher target specified through + // an attribute or flag. We wish to retain the runfiles of the default launcher, *except* + // for the original cc_binary artifact, because we've swapped it out with our custom + // launcher. Hence, instead of calling builder.addTarget(), or adding an odd method + // to Runfiles.Builder, we "unravel" the call and manually add things to the builder. + // Because the NestedSet representing each target's launcher runfiles is re-built here, + // we may see increased memory consumption for representing the target's runfiles. + Runfiles runfiles = + defaultLauncher.getProvider(RunfilesProvider.class) + .getDefaultRunfiles(); + NestedSetBuilder<Artifact> unconditionalArtifacts = NestedSetBuilder.compileOrder(); + for (Artifact a : runfiles.getUnconditionalArtifacts()) { + if (!a.equals(defaultLauncherArtifact)) { + unconditionalArtifacts.add(a); + } + } + builder.addTransitiveArtifacts(unconditionalArtifacts.build()); + builder.addSymlinks(runfiles.getSymlinks()); + builder.addRootSymlinks(runfiles.getRootSymlinks()); + builder.addPruningManifests(runfiles.getPruningManifests()); + } else { + builder.addTarget(defaultLauncher, RunfilesProvider.DEFAULT_RUNFILES); + } + } + + semantics.addRunfilesForBinary(ruleContext, launcher, builder); + builder.addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES); + builder.add(ruleContext, JavaRunfilesProvider.TO_RUNFILES); + + List<? extends TransitiveInfoCollection> runtimeDeps = + ruleContext.getPrerequisites("runtime_deps", Mode.TARGET); + builder.addTargets(runtimeDeps, JavaRunfilesProvider.TO_RUNFILES); + builder.addTargets(runtimeDeps, RunfilesProvider.DEFAULT_RUNFILES); + semantics.addDependenciesForRunfiles(ruleContext, builder); + + if (ruleContext.getConfiguration().isCodeCoverageEnabled()) { + Artifact instrumentedJar = common.getJavaCompilationArtifacts().getInstrumentedJar(); + if (instrumentedJar != null) { + builder.addArtifact(instrumentedJar); + } + } + + builder.addArtifacts((Iterable<Artifact>) common.getRuntimeClasspath()); + + // Add the JDK files if it comes from the source repository (see java_stub_template.txt). + TransitiveInfoCollection javabaseTarget = ruleContext.getPrerequisite(":jvm", Mode.HOST); + if (javabaseTarget != null) { + builder.addArtifacts( + (Iterable<Artifact>) javabaseTarget.getProvider(FileProvider.class).getFilesToBuild()); + + // Add symlinks to the C++ runtime libraries under a path that can be built + // into the Java binary without having to embed the crosstool, gcc, and grte + // version information contained within the libraries' package paths. + for (Artifact lib : dynamicRuntimeActionInputs) { + PathFragment path = CPP_RUNTIMES.getRelative(lib.getExecPath().getBaseName()); + builder.addSymlink(path, lib); + } + } + } + + private NestedSet<Artifact> collectTransitiveSourceJars(JavaCommon common, Artifact srcJar) { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + + builder.add(srcJar); + for (JavaSourceJarsProvider dep : common.getDependencies(JavaSourceJarsProvider.class)) { + builder.addTransitive(dep.getTransitiveSourceJars()); + } + return builder.build(); + } + + /** + * Collects the native libraries in the transitive closure of the deps. + * + * @param deps the dependencies to be included as roots of the transitive closure. + * @return the native libraries found in the transitive closure of the deps. + */ + public static Collection<Artifact> collectNativeLibraries( + Iterable<? extends TransitiveInfoCollection> deps) { + NestedSet<LinkerInput> linkerInputs = new NativeLibraryNestedSetBuilder() + .addJavaTargets(deps) + .build(); + ImmutableList.Builder<Artifact> result = ImmutableList.builder(); + for (LinkerInput linkerInput : linkerInputs) { + result.add(linkerInput.getArtifact()); + } + + return result.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaBuildInfoFactory.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBuildInfoFactory.java new file mode 100644 index 0000000000..442b85bcf9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBuildInfoFactory.java @@ -0,0 +1,145 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoCollection; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.rules.java.WriteBuildInfoPropertiesAction.TimestampFormatter; +import com.google.devtools.build.lib.vfs.PathFragment; + +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.util.ArrayList; +import java.util.List; + +/** + * Java build info creation - generates properties file that contain the corresponding build-info + * data. + */ +public abstract class JavaBuildInfoFactory implements BuildInfoFactory { + public static final BuildInfoKey KEY = new BuildInfoKey("Java"); + + static final PathFragment BUILD_INFO_NONVOLATILE_PROPERTIES_NAME = + new PathFragment("build-info-nonvolatile.properties"); + static final PathFragment BUILD_INFO_VOLATILE_PROPERTIES_NAME = + new PathFragment("build-info-volatile.properties"); + static final PathFragment BUILD_INFO_REDACTED_PROPERTIES_NAME = + new PathFragment("build-info-redacted.properties"); + + private static final DateTimeFormatter DEFAULT_TIME_FORMAT = + DateTimeFormat.forPattern("EEE MMM d HH:mm:ss yyyy"); + + // A default formatter that returns a date in UTC format. + private static final TimestampFormatter DEFAULT_FORMATTER = new TimestampFormatter() { + @Override + public String format(long timestamp) { + return new DateTime(timestamp, DateTimeZone.UTC).toString(DEFAULT_TIME_FORMAT) + " (" + + timestamp / 1000 + ')'; + } + }; + + @Override + public final BuildInfoCollection create(BuildInfoContext context, BuildConfiguration config, + Artifact stableStatus, Artifact volatileStatus) { + WriteBuildInfoPropertiesAction redactedInfo = getHeader(context, + config, + BUILD_INFO_REDACTED_PROPERTIES_NAME, + Artifact.NO_ARTIFACTS, + createRedactedTranslator(), + true, + true); + WriteBuildInfoPropertiesAction nonvolatileInfo = getHeader(context, + config, + BUILD_INFO_NONVOLATILE_PROPERTIES_NAME, + ImmutableList.of(stableStatus), + createNonVolatileTranslator(), + false, + true); + WriteBuildInfoPropertiesAction volatileInfo = getHeader(context, + config, + BUILD_INFO_VOLATILE_PROPERTIES_NAME, + ImmutableList.of(volatileStatus), + createVolatileTranslator(), + true, + false); + List<Action> actions = new ArrayList<Action>(3); + actions.add(redactedInfo); + actions.add(nonvolatileInfo); + actions.add(volatileInfo); + return new BuildInfoCollection(actions, + ImmutableList.of(nonvolatileInfo.getPrimaryOutput(), volatileInfo.getPrimaryOutput()), + ImmutableList.of(redactedInfo.getPrimaryOutput())); + } + + /** + * Creates a {@link BuildInfoPropertiesTranslator} to use for volatile keys. + */ + protected abstract BuildInfoPropertiesTranslator createVolatileTranslator(); + + /** + * Creates a {@link BuildInfoPropertiesTranslator} to use for non-volatile keys. + */ + protected abstract BuildInfoPropertiesTranslator createNonVolatileTranslator(); + + /** + * Creates a {@link BuildInfoPropertiesTranslator} to use for redacted version of the build + * informations. + */ + protected abstract BuildInfoPropertiesTranslator createRedactedTranslator(); + + /** + * Specifies the {@link TimestampFormatter} to use to output dates in the properties file. + */ + protected TimestampFormatter getTimestampFormatter() { + return DEFAULT_FORMATTER; + } + + private WriteBuildInfoPropertiesAction getHeader(BuildInfoContext context, + BuildConfiguration config, + PathFragment propertyFileName, + ImmutableList<Artifact> inputs, + BuildInfoPropertiesTranslator translator, + boolean includeVolatile, + boolean includeNonVolatile) { + Root outputPath = config.getIncludeDirectory(); + final Artifact output = context.getBuildInfoArtifact(propertyFileName, outputPath, + includeVolatile && !inputs.isEmpty() ? BuildInfoType.NO_REBUILD + : BuildInfoType.FORCE_REBUILD_IF_CHANGED); + return new WriteBuildInfoPropertiesAction(inputs, + output, + translator, + includeVolatile, + includeNonVolatile, + getTimestampFormatter()); + } + + @Override + public final BuildInfoKey getKey() { + return KEY; + } + + @Override + public boolean isEnabled(BuildConfiguration config) { + return config.hasFragment(JavaConfiguration.class); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCcLinkParamsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCcLinkParamsProvider.java new file mode 100644 index 0000000000..5218f3fa2c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCcLinkParamsProvider.java @@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore.CcLinkParamsStoreImpl; + +/** + * A target that provides C++ libraries to be linked into Java targets. + */ +@Immutable +public final class JavaCcLinkParamsProvider implements TransitiveInfoProvider { + private final CcLinkParamsStoreImpl store; + + public JavaCcLinkParamsProvider(CcLinkParamsStore store) { + this.store = new CcLinkParamsStoreImpl(store); + } + + public CcLinkParamsStore getLinkParams() { + return store; + } + + public static final Function<TransitiveInfoCollection, CcLinkParamsStore> TO_LINK_PARAMS = + new Function<TransitiveInfoCollection, CcLinkParamsStore>() { + @Override + public CcLinkParamsStore apply(TransitiveInfoCollection input) { + JavaCcLinkParamsProvider provider = input.getProvider( + JavaCcLinkParamsProvider.class); + return provider == null ? null : provider.getLinkParams(); + } + }; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java new file mode 100644 index 0000000000..c55a74e0bb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java @@ -0,0 +1,651 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToCompileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.Util; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CppCompilationContext; +import com.google.devtools.build.lib.rules.cpp.LinkerInput; +import com.google.devtools.build.lib.rules.java.DirectDependencyProvider.Dependency; +import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType; +import com.google.devtools.build.lib.rules.test.BaselineCoverageAction; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.LocalMetadataCollector; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProviderImpl; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A helper class to create configured targets for Java rules. + */ +public class JavaCommon { + private static final Function<TransitiveInfoCollection, Label> GET_COLLECTION_LABEL = + new Function<TransitiveInfoCollection, Label>() { + @Override + public Label apply(TransitiveInfoCollection collection) { + return collection.getLabel(); + } + }; + + /** + * Collects all metadata files generated by Java compilation actions. + */ + private static final LocalMetadataCollector JAVA_METADATA_COLLECTOR = + new LocalMetadataCollector() { + @Override + public void collectMetadataArtifacts(Iterable<Artifact> objectFiles, + AnalysisEnvironment analysisEnvironment, NestedSetBuilder<Artifact> metadataFilesBuilder) { + for (Artifact artifact : objectFiles) { + Action action = analysisEnvironment.getLocalGeneratingAction(artifact); + if (action instanceof JavaCompileAction) { + addOutputs(metadataFilesBuilder, action, JavaSemantics.COVERAGE_METADATA); + } + } + } + }; + + private ClasspathConfiguredFragment classpathFragment = new ClasspathConfiguredFragment(); + private JavaCompilationArtifacts javaArtifacts = JavaCompilationArtifacts.EMPTY; + private ImmutableList<String> javacOpts; + + // Targets treated as deps in compilation time, runtime time and both + private final ImmutableMap<ClasspathType, ImmutableList<TransitiveInfoCollection>> + targetsTreatedAsDeps; + + private ImmutableList<Artifact> sources = ImmutableList.of(); + private ImmutableList<JavaPluginInfoProvider> activePlugins = ImmutableList.of(); + + private final RuleContext ruleContext; + private final JavaSemantics semantics; + + public JavaCommon(RuleContext ruleContext, JavaSemantics semantics) { + this(ruleContext, semantics, + collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.COMPILE_ONLY), + collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.RUNTIME_ONLY), + collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.BOTH)); + } + + public JavaCommon(RuleContext ruleContext, + JavaSemantics semantics, + ImmutableList<TransitiveInfoCollection> compileDeps, + ImmutableList<TransitiveInfoCollection> runtimeDeps, + ImmutableList<TransitiveInfoCollection> bothDeps) { + this.ruleContext = ruleContext; + this.semantics = semantics; + this.targetsTreatedAsDeps = ImmutableMap.of( + ClasspathType.COMPILE_ONLY, compileDeps, + ClasspathType.RUNTIME_ONLY, runtimeDeps, + ClasspathType.BOTH, bothDeps); + } + + public void setClassPathFragment(ClasspathConfiguredFragment classpathFragment) { + this.classpathFragment = classpathFragment; + } + + public void setJavaCompilationArtifacts(JavaCompilationArtifacts javaArtifacts) { + this.javaArtifacts = javaArtifacts; + } + + public JavaCompilationArtifacts getJavaCompilationArtifacts() { + return javaArtifacts; + } + + public ImmutableList<Artifact> getProcessorClasspathJars() { + Set<Artifact> processorClasspath = new LinkedHashSet<>(); + for (JavaPluginInfoProvider plugin : activePlugins) { + for (Artifact classpathJar : plugin.getProcessorClasspath()) { + processorClasspath.add(classpathJar); + } + } + return ImmutableList.copyOf(processorClasspath); + } + + public ImmutableList<String> getProcessorClassNames() { + Set<String> processorNames = new LinkedHashSet<>(); + for (JavaPluginInfoProvider plugin : activePlugins) { + processorNames.addAll(plugin.getProcessorClasses()); + } + return ImmutableList.copyOf(processorNames); + } + + /** + * Creates the java.library.path from a list of the native libraries. + * Concatenates the parent directories of the shared libraries into a Java + * search path. Each relative path entry is prepended with "${JAVA_RUNFILES}/" + * so it can be resolved at runtime. + * + * @param sharedLibraries a collection of native libraries to create the java + * library path from + * @return a String containing the ":" separated java library path + */ + public static String javaLibraryPath(Collection<Artifact> sharedLibraries) { + StringBuilder buffer = new StringBuilder(); + Set<PathFragment> entries = new HashSet<>(); + for (Artifact sharedLibrary : sharedLibraries) { + PathFragment entry = sharedLibrary.getRootRelativePath().getParentDirectory(); + if (entries.add(entry)) { + if (buffer.length() > 0) { + buffer.append(':'); + } + buffer.append("${JAVA_RUNFILES}/" + Constants.RUNFILES_PREFIX + "/"); + buffer.append(entry.getPathString()); + } + } + return buffer.toString(); + } + + /** + * Collects Java compilation arguments for this target. + * + * @param recursive Whether to scan dependencies recursively. + * @param isNeverLink Whether the target has the 'neverlink' attr. + */ + JavaCompilationArgs collectJavaCompilationArgs(boolean recursive, boolean isNeverLink, + Iterable<SourcesJavaCompilationArgsProvider> compilationArgsFromSources) { + ClasspathType type = isNeverLink ? ClasspathType.COMPILE_ONLY : ClasspathType.BOTH; + JavaCompilationArgs.Builder builder = JavaCompilationArgs.builder() + .merge(getJavaCompilationArtifacts(), isNeverLink) + .addTransitiveTargets(getExports(ruleContext), recursive, type); + if (recursive) { + builder + .addTransitiveTargets(targetsTreatedAsDeps(ClasspathType.COMPILE_ONLY), recursive, type) + .addTransitiveTargets(getRuntimeDeps(ruleContext), recursive, ClasspathType.RUNTIME_ONLY) + .addSourcesTransitiveCompilationArgs(compilationArgsFromSources, recursive, type); + } + return builder.build(); + } + + /** + * Collects Java dependency artifacts for this target. + * + * @param outDeps output (compile-time) dependency artifact of this target + */ + NestedSet<Artifact> collectCompileTimeDependencyArtifacts(Artifact outDeps) { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + if (outDeps != null) { + builder.add(outDeps); + } + + for (JavaCompilationArgsProvider provider : AnalysisUtils.getProviders( + getExports(ruleContext), JavaCompilationArgsProvider.class)) { + builder.addTransitive(provider.getCompileTimeJavaDependencyArtifacts()); + } + return builder.build(); + } + + public static List<TransitiveInfoCollection> getExports(RuleContext ruleContext) { + // We need to check here because there are classes inheriting from this class that implement + // rules that don't have this attribute. + if (ruleContext.getRule().getRuleClassObject().hasAttr("exports", Type.LABEL_LIST)) { + return ImmutableList.copyOf(ruleContext.getPrerequisites("exports", Mode.TARGET)); + } else { + return ImmutableList.of(); + } + } + + /** + * Sanity checks the given runtime dependencies, and emits errors if there is a problem. + * Also called by {@link #initCommon()} for the current target's runtime dependencies. + */ + public void checkRuntimeDeps(List<TransitiveInfoCollection> runtimeDepInfo) { + for (TransitiveInfoCollection c : runtimeDepInfo) { + JavaNeverlinkInfoProvider neverLinkedness = + c.getProvider(JavaNeverlinkInfoProvider.class); + if (neverLinkedness == null) { + continue; + } + boolean reportError = !ruleContext.getConfiguration().getAllowRuntimeDepsOnNeverLink(); + if (neverLinkedness.isNeverlink()) { + String msg = String.format("neverlink dep %s not allowed in runtime deps", c.getLabel()); + if (reportError) { + ruleContext.attributeError("runtime_deps", msg); + } else { + ruleContext.attributeWarning("runtime_deps", msg); + } + } + } + } + + /** + * Returns transitive Java native libraries. + * + * @see JavaNativeLibraryProvider + */ + protected NestedSet<LinkerInput> collectTransitiveJavaNativeLibraries() { + NativeLibraryNestedSetBuilder builder = new NativeLibraryNestedSetBuilder(); + builder.addJavaTargets(targetsTreatedAsDeps(ClasspathType.BOTH)); + + if (ruleContext.getRule().isAttrDefined("data", Type.LABEL_LIST)) { + builder.addJavaTargets(ruleContext.getPrerequisites("data", Mode.DATA)); + } + return builder.build(); + } + + /** + * Collects transitive source jars for the current rule. + * + * @param targetSrcJar The source jar artifact corresponding to the output of the current rule. + * @return A nested set containing all of the source jar artifacts on which the current rule + * transitively depends. + */ + public NestedSet<Artifact> collectTransitiveSourceJars(Artifact targetSrcJar) { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + + builder.add(targetSrcJar); + for (JavaSourceJarsProvider dep : getDependencies(JavaSourceJarsProvider.class)) { + builder.addTransitive(dep.getTransitiveSourceJars()); + } + return builder.build(); + } + + /** + * Collects transitive C++ dependencies. + */ + protected CppCompilationContext collectTransitiveCppDeps() { + CppCompilationContext.Builder builder = new CppCompilationContext.Builder(ruleContext); + for (TransitiveInfoCollection dep : targetsTreatedAsDeps(ClasspathType.BOTH)) { + CppCompilationContext context = + dep.getProvider(CppCompilationContext.class); + if (context != null) { + builder.mergeDependentContext(context); + } + } + return builder.build(); + } + + /** + * Collects labels of targets and artifacts reached transitively via the "exports" attribute. + */ + protected NestedSet<Label> collectTransitiveExports() { + NestedSetBuilder<Label> builder = NestedSetBuilder.stableOrder(); + List<TransitiveInfoCollection> currentRuleExports = getExports(ruleContext); + + builder.addAll(Iterables.transform(currentRuleExports, GET_COLLECTION_LABEL)); + + for (TransitiveInfoCollection dep : currentRuleExports) { + JavaExportsProvider exportsProvider = dep.getProvider(JavaExportsProvider.class); + + if (exportsProvider != null) { + builder.addTransitive(exportsProvider.getTransitiveExports()); + } + } + + return builder.build(); + } + + public final void initializeJavacOpts() { + initializeJavacOpts(semantics.getExtraJavacOpts(ruleContext)); + } + + public final void initializeJavacOpts(Iterable<String> extraJavacOpts) { + javacOpts = ImmutableList.copyOf(Iterables.concat( + JavaToolchainProvider.getDefaultJavacOptions(ruleContext), + ruleContext.getTokenizedStringListAttr("javacopts"), extraJavacOpts)); + } + + /** + * Returns the string that the stub should use to determine the JVM + * @param launcher if non-null, the cc_binary used to launch the Java Virtual Machine + */ + public String getJavaBinSubstitution(@Nullable Artifact launcher) { + PathFragment javaExecutable; + + if (launcher != null) { + javaExecutable = launcher.getRootRelativePath(); + } else { + javaExecutable = ruleContext.getFragment(Jvm.class).getJavaExecutable(); + } + + String pathPrefix = + javaExecutable.isAbsolute() ? "" : "${JAVA_RUNFILES}/" + Constants.RUNFILES_PREFIX + "/"; + return "JAVABIN=${JAVABIN:-" + pathPrefix + javaExecutable.getPathString() + "}"; + } + + /** + * Heuristically determines the name of the primary Java class for this + * executable, based on the rule name and the "srcs" list. + * + * <p>(This is expected to be the class containing the "main" method for a + * java_binary, or a JUnit Test class for a java_test.) + * + * @param sourceFiles the source files for this rule + * @return a fully qualified Java class name, or null if none could be + * determined. + */ + public String determinePrimaryClass(Collection<Artifact> sourceFiles) { + if (!sourceFiles.isEmpty()) { + String mainSource = ruleContext.getTarget().getName() + ".java"; + for (Artifact sourceFile : sourceFiles) { + PathFragment path = sourceFile.getRootRelativePath(); + if (path.getBaseName().equals(mainSource)) { + return JavaUtil.getJavaFullClassname(FileSystemUtils.removeExtension(path)); + } + } + } + // Last resort: Use the name and package name of the target. + // TODO(bazel-team): this should be fixed to use a source file from the dependencies to + // determine the package of the Java class. + return JavaUtil.getJavaFullClassname(Util.getWorkspaceRelativePath(ruleContext.getTarget())); + } + + /** + * Gets the value of the "jvm_flags" attribute combining it with the default + * options and expanding any make variables. + */ + public List<String> getJvmFlags() { + List<String> jvmFlags = new ArrayList<>(); + jvmFlags.addAll(ruleContext.getFragment(JavaConfiguration.class).getDefaultJvmFlags()); + jvmFlags.addAll(ruleContext.expandedMakeVariablesList("jvm_flags")); + return jvmFlags; + } + + private static List<TransitiveInfoCollection> getRuntimeDeps(RuleContext ruleContext) { + // We need to check here because there are classes inheriting from this class that implement + // rules that don't have this attribute. + if (ruleContext.getRule().getRuleClassObject().hasAttr("runtime_deps", Type.LABEL_LIST)) { + return ImmutableList.copyOf(ruleContext.getPrerequisites("runtime_deps", Mode.TARGET)); + } else { + return ImmutableList.of(); + } + } + + public JavaTargetAttributes.Builder initCommon() { + return initCommon(Collections.<Artifact>emptySet()); + } + + /** + * Initialize the common actions and build various collections of artifacts + * for the initializationHook() methods of the subclasses. + * + * <p>Note that not all subclasses call this method. + * + * @return the processed attributes + */ + public JavaTargetAttributes.Builder initCommon(Collection<Artifact> extraSrcs) { + Preconditions.checkState(javacOpts != null); + sources = ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list(); + activePlugins = collectPlugins(); + + JavaTargetAttributes.Builder javaTargetAttributes = new JavaTargetAttributes.Builder(semantics); + processSrcs(javaTargetAttributes, javacOpts); + javaTargetAttributes.addSourceArtifacts(extraSrcs); + processRuntimeDeps(javaTargetAttributes); + + semantics.commonDependencyProcessing(ruleContext, javaTargetAttributes, + targetsTreatedAsDeps(ClasspathType.COMPILE_ONLY)); + + // Check that we have do not have both sources and jars. + if ((javaTargetAttributes.hasSourceFiles() || javaTargetAttributes.hasSourceJars()) + && javaTargetAttributes.hasJarFiles()) { + ruleContext.attributeWarning("srcs", "cannot use both Java sources - source " + + "jars or source files - and precompiled jars"); + } + + if (disallowDepsWithoutSrcs(ruleContext.getRule().getRuleClass()) + && ruleContext.attributes().get("srcs", Type.LABEL_LIST).isEmpty() + && ruleContext.getRule().isAttributeValueExplicitlySpecified("deps")) { + ruleContext.attributeError("deps", "deps not allowed without srcs; move to runtime_deps?"); + } + + javaTargetAttributes.addResources(semantics.collectResources(ruleContext)); + addPlugins(javaTargetAttributes); + + javaTargetAttributes.setRuleKind(ruleContext.getRule().getRuleClass()); + javaTargetAttributes.setTargetLabel(ruleContext.getLabel()); + + return javaTargetAttributes; + } + + private boolean disallowDepsWithoutSrcs(String ruleClass) { + return ruleClass.equals("java_library") + || ruleClass.equals("java_binary") + || ruleClass.equals("java_test"); + } + + public ImmutableList<? extends TransitiveInfoCollection> targetsTreatedAsDeps( + ClasspathType type) { + return targetsTreatedAsDeps.get(type); + } + + private static ImmutableList<TransitiveInfoCollection> collectTargetsTreatedAsDeps( + RuleContext ruleContext, JavaSemantics semantics, ClasspathType type) { + ImmutableList.Builder<TransitiveInfoCollection> builder = new Builder<>(); + + if (!type.equals(ClasspathType.COMPILE_ONLY)) { + builder.addAll(getRuntimeDeps(ruleContext)); + builder.addAll(getExports(ruleContext)); + } + builder.addAll(ruleContext.getPrerequisites("deps", Mode.TARGET)); + + semantics.collectTargetsTreatedAsDeps(ruleContext, builder); + + // Implicitly add dependency on java launcher cc_binary when --java_launcher= is enabled, + // or when launcher attribute is specified in a build rule. + TransitiveInfoCollection launcher = JavaHelper.launcherForTarget(semantics, ruleContext); + if (launcher != null) { + builder.add(launcher); + } + + return builder.build(); + } + + public void addTransitiveInfoProviders(RuleConfiguredTargetBuilder builder, + NestedSet<Artifact> filesToBuild, @Nullable Artifact classJar) { + InstrumentedFilesCollector instrumentedFilesCollector = + new InstrumentedFilesCollector(ruleContext, semantics.getCoverageInstrumentationSpec(), + JAVA_METADATA_COLLECTOR, filesToBuild); + + builder + .add(InstrumentedFilesProvider.class, new InstrumentedFilesProviderImpl( + instrumentedFilesCollector)) + .add(FilesToCompileProvider.class, + new FilesToCompileProvider(getFilesToCompile(classJar))) + .add(JavaExportsProvider.class, new JavaExportsProvider(collectTransitiveExports())); + + if (!TargetUtils.isTestRule(ruleContext.getTarget())) { + ImmutableList<Artifact> baselineCoverageArtifacts = + BaselineCoverageAction.getBaselineCoverageArtifacts(ruleContext, + instrumentedFilesCollector.getInstrumentedFiles()); + builder.setBaselineCoverageArtifacts(baselineCoverageArtifacts); + } + } + + /** + * Processes the sources of this target, adding them as messages, proper + * sources or to the list of targets treated as deps as required. + */ + private void processSrcs(JavaTargetAttributes.Builder attributes, + ImmutableList<String> javacOpts) { + for (MessageBundleProvider srcItem : ruleContext.getPrerequisites( + "srcs", Mode.TARGET, MessageBundleProvider.class)) { + attributes.addMessages(srcItem.getMessages()); + } + + attributes.addSourceArtifacts(sources); + + addCompileTimeClassPathEntriesMaybeThroughIjar(attributes, javacOpts); + } + + /** + * Processes the transitive runtime_deps of this target. + */ + private void processRuntimeDeps(JavaTargetAttributes.Builder attributes) { + List<TransitiveInfoCollection> runtimeDepInfo = getRuntimeDeps(ruleContext); + checkRuntimeDeps(runtimeDepInfo); + JavaCompilationArgs args = JavaCompilationArgs.builder() + .addTransitiveTargets(runtimeDepInfo, true, ClasspathType.RUNTIME_ONLY) + .build(); + attributes.addRuntimeClassPathEntries(args.getRuntimeJars()); + attributes.addInstrumentationMetadataEntries(args.getInstrumentationMetadata()); + } + + public Iterable<SourcesJavaCompilationArgsProvider> compilationArgsFromSources() { + return ruleContext.getPrerequisites("srcs", Mode.TARGET, + SourcesJavaCompilationArgsProvider.class); + } + + /** + * Adds jars in the given group of entries to the compile time classpath after + * using ijar to create jar interfaces for the generated jars. + */ + private void addCompileTimeClassPathEntriesMaybeThroughIjar( + JavaTargetAttributes.Builder attributes, ImmutableList<String> javacOpts) { + JavaCompilationHelper helper = new JavaCompilationHelper( + ruleContext, semantics, javacOpts, attributes); + for (FileProvider provider : ruleContext + .getPrerequisites("srcs", Mode.TARGET, FileProvider.class)) { + Iterable<Artifact> jarFiles = helper.filterGeneratedJarsThroughIjar( + FileType.filter(provider.getFilesToBuild(), JavaSemantics.JAR)); + List<Artifact> jarsWithOwners = Lists.newArrayList(jarFiles); + attributes.addDirectCompileTimeClassPathEntries(jarsWithOwners); + attributes.addCompileTimeJarFiles(jarsWithOwners); + } + } + + /** + * Adds information about the annotation processors that should be run for this java target to + * the target attributes. + */ + private void addPlugins(JavaTargetAttributes.Builder attributes) { + for (JavaPluginInfoProvider plugin : activePlugins) { + for (String name : plugin.getProcessorClasses()) { + attributes.addProcessorName(name); + } + // Now get the plugin-libraries runtime classpath. + attributes.addProcessorPath(plugin.getProcessorClasspath()); + } + } + + private ImmutableList<JavaPluginInfoProvider> collectPlugins() { + List<JavaPluginInfoProvider> result = new ArrayList<>(); + Iterables.addAll(result, getPluginInfoProvidersForAttribute(":java_plugins", Mode.HOST)); + Iterables.addAll(result, getPluginInfoProvidersForAttribute("plugins", Mode.HOST)); + Iterables.addAll(result, getPluginInfoProvidersForAttribute("deps", Mode.TARGET)); + return ImmutableList.copyOf(result); + } + + Iterable<JavaPluginInfoProvider> getPluginInfoProvidersForAttribute(String attribute, + Mode mode) { + if (ruleContext.getRule().getRuleClassObject().hasAttr(attribute, Type.LABEL_LIST)) { + return ruleContext.getPrerequisites(attribute, mode, JavaPluginInfoProvider.class); + } + return ImmutableList.of(); + } + + /** + * Gets all the deps. + */ + public final Iterable<? extends TransitiveInfoCollection> getDependencies() { + return targetsTreatedAsDeps(ClasspathType.BOTH); + } + + /** + * Gets all the deps that implement a particular provider. + */ + public final <P extends TransitiveInfoProvider> Iterable<P> getDependencies( + Class<P> provider) { + return AnalysisUtils.getProviders(getDependencies(), provider); + } + + /** + * Returns true if and only if this target has the neverlink attribute set to + * 1, or false if the neverlink attribute does not exist (for example, on + * *_binary targets) + * + * @return the value of the neverlink attribute. + */ + public final boolean isNeverLink() { + return ruleContext.getRule().isAttrDefined("neverlink", Type.BOOLEAN) && + ruleContext.attributes().get("neverlink", Type.BOOLEAN); + } + + private ImmutableList<Artifact> getFilesToCompile(Artifact classJar) { + if (classJar == null) { + // Some subclasses don't produce jars + return ImmutableList.of(); + } + return ImmutableList.of(classJar); + } + + public ImmutableList<Dependency> computeStrictDepsFromJavaAttributes( + JavaTargetAttributes javaTargetAttributes) { + Multimap<Label, String> depMap = HashMultimap.<Label, String>create(); + for (Artifact jar : javaTargetAttributes.getDirectJars()) { + depMap.put(Preconditions.checkNotNull(jar.getOwner()), + jar.getExecPathString()); + } + ImmutableList.Builder<Dependency> depOuts = ImmutableList.builder(); + for (Label label : depMap.keySet()) { + depOuts.add(new Dependency(label, depMap.get(label))); + } + return depOuts.build(); + } + + public ImmutableList<Artifact> getSrcsArtifacts() { + return sources; + } + + public ImmutableList<String> getJavacOpts() { + return javacOpts; + } + + public ImmutableList<Artifact> getBootClasspath() { + return classpathFragment.getBootClasspath(); + } + + public NestedSet<Artifact> getRuntimeClasspath() { + return classpathFragment.getRuntimeClasspath(); + } + + public NestedSet<Artifact> getCompileTimeClasspath() { + return classpathFragment.getCompileTimeClasspath(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgs.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgs.java new file mode 100644 index 0000000000..145d6467de --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgs.java @@ -0,0 +1,301 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.util.FileType; + +import java.util.Collection; + +/** + * A container of Java compilation artifacts. + */ +public final class JavaCompilationArgs { + // TODO(bazel-team): It would be desirable to use LinkOrderNestedSet here so that + // parents-before-deps is preserved for graphs that are not trees. However, the legacy + // JavaLibraryCollector implemented naive link ordering and many targets in the + // depot depend on the consistency of left-to-right ordering that is not provided by + // LinkOrderNestedSet. They simply list their local dependencies before + // other targets that may use conflicting dependencies, and the local deps + // appear earlier on the classpath, as desired. Behavior of LinkOrderNestedSet + // can be very unintuitive in case of conflicting orders, because the order is + // decided by the rightmost branch in such cases. For example, if A depends on {junit4, + // B}, B depends on {C, D}, C depends on {junit3}, and D depends on {junit4}, + // the classpath of A will have junit3 before junit4. + private final NestedSet<Artifact> runtimeJars; + private final NestedSet<Artifact> compileTimeJars; + private final NestedSet<Artifact> instrumentationMetadata; + + public static final JavaCompilationArgs EMPTY_ARGS = new JavaCompilationArgs( + NestedSetBuilder.<Artifact>create(Order.NAIVE_LINK_ORDER), + NestedSetBuilder.<Artifact>create(Order.NAIVE_LINK_ORDER), + NestedSetBuilder.<Artifact>create(Order.NAIVE_LINK_ORDER)); + + private JavaCompilationArgs(NestedSet<Artifact> runtimeJars, + NestedSet<Artifact> compileTimeJars, + NestedSet<Artifact> instrumentationMetadata) { + this.runtimeJars = runtimeJars; + this.compileTimeJars = compileTimeJars; + this.instrumentationMetadata = instrumentationMetadata; + } + + /** + * Returns transitive runtime jars. + */ + public NestedSet<Artifact> getRuntimeJars() { + return runtimeJars; + } + + /** + * Returns transitive compile-time jars. + */ + public NestedSet<Artifact> getCompileTimeJars() { + return compileTimeJars; + } + + /** + * Returns transitive instrumentation metadata jars. + */ + public NestedSet<Artifact> getInstrumentationMetadata() { + return instrumentationMetadata; + } + + /** + * Returns a new builder instance. + */ + public static final Builder builder() { + return new Builder(); + } + + /** + * Builder for {@link JavaCompilationArgs}. + * + * + */ + public static final class Builder { + private final NestedSetBuilder<Artifact> runtimeJarsBuilder = + NestedSetBuilder.naiveLinkOrder(); + private final NestedSetBuilder<Artifact> compileTimeJarsBuilder = + NestedSetBuilder.naiveLinkOrder(); + private final NestedSetBuilder<Artifact> instrumentationMetadataBuilder = + NestedSetBuilder.naiveLinkOrder(); + + /** + * Use {@code TransitiveJavaCompilationArgs#builder()} to instantiate the builder. + */ + private Builder() { + } + + /** + * Legacy method for dealing with objects which construct + * {@link JavaCompilationArtifacts} objects. + */ + // TODO(bazel-team): Remove when we get rid of JavaCompilationArtifacts. + public Builder merge(JavaCompilationArtifacts other, boolean isNeverLink) { + if (!isNeverLink) { + addRuntimeJars(other.getRuntimeJars()); + } + addCompileTimeJars(other.getCompileTimeJars()); + addInstrumentationMetadata(other.getInstrumentationMetadata()); + return this; + } + + /** + * Legacy method for dealing with objects which construct + * {@link JavaCompilationArtifacts} objects. + */ + public Builder merge(JavaCompilationArtifacts other) { + return merge(other, false); + } + + public Builder addRuntimeJar(Artifact runtimeJar) { + this.runtimeJarsBuilder.add(runtimeJar); + return this; + } + + public Builder addRuntimeJars(Iterable<Artifact> runtimeJars) { + this.runtimeJarsBuilder.addAll(runtimeJars); + return this; + } + + public Builder addCompileTimeJar(Artifact compileTimeJar) { + this.compileTimeJarsBuilder.add(compileTimeJar); + return this; + } + + public Builder addCompileTimeJars(Iterable<Artifact> compileTimeJars) { + this.compileTimeJarsBuilder.addAll(compileTimeJars); + return this; + } + + public Builder addInstrumentationMetadata(Artifact instrumentationMetadata) { + this.instrumentationMetadataBuilder.add(instrumentationMetadata); + return this; + } + + public Builder addInstrumentationMetadata(Collection<Artifact> instrumentationMetadata) { + this.instrumentationMetadataBuilder.addAll(instrumentationMetadata); + return this; + } + + public Builder addTransitiveCompilationArgs( + JavaCompilationArgsProvider dep, boolean recursive, ClasspathType type) { + JavaCompilationArgs args = recursive + ? dep.getRecursiveJavaCompilationArgs() + : dep.getJavaCompilationArgs(); + addTransitiveArgs(args, type); + return this; + } + + public Builder addTransitiveCompilationArgs( + SourcesJavaCompilationArgsProvider dep, boolean recursive, ClasspathType type) { + JavaCompilationArgs args; + if (recursive) { + args = dep.getRecursiveJavaCompilationArgs(); + } else { + args = dep.getJavaCompilationArgs(); + } + addTransitiveArgs(args, type); + return this; + } + + public Builder addSourcesTransitiveCompilationArgs( + Iterable<? extends SourcesJavaCompilationArgsProvider> deps, + boolean recursive, + ClasspathType type) { + for (SourcesJavaCompilationArgsProvider dep : deps) { + addTransitiveCompilationArgs(dep, recursive, type); + } + + return this; + } + + /** + * Merges the artifacts of another target. + */ + public Builder addTransitiveTarget(TransitiveInfoCollection dep, boolean recursive, + ClasspathType type) { + JavaCompilationArgsProvider provider = dep.getProvider(JavaCompilationArgsProvider.class); + if (provider != null) { + addTransitiveCompilationArgs(provider, recursive, type); + return this; + } else { + NestedSet<Artifact> filesToBuild = + dep.getProvider(FileProvider.class).getFilesToBuild(); + for (Artifact jar : FileType.filter(filesToBuild, JavaSemantics.JAR)) { + addCompileTimeJar(jar); + addRuntimeJar(jar); + } + } + return this; + } + + /** + * Merges the artifacts of a collection of targets. + */ + public Builder addTransitiveTargets(Iterable<? extends TransitiveInfoCollection> deps, + boolean recursive, ClasspathType type) { + for (TransitiveInfoCollection dep : deps) { + addTransitiveTarget(dep, recursive, type); + } + return this; + } + + /** + * Merges the artifacts of a collection of targets. + */ + public Builder addTransitiveTargets(Iterable<? extends TransitiveInfoCollection> deps, + boolean recursive) { + return addTransitiveTargets(deps, recursive, ClasspathType.BOTH); + } + + /** + * Merges the artifacts of a collection of targets. + */ + public Builder addTransitiveDependencies(Iterable<JavaCompilationArgsProvider> deps, + boolean recursive) { + for (JavaCompilationArgsProvider dep : deps) { + addTransitiveDependency(dep, recursive, ClasspathType.BOTH); + } + return this; + } + + /** + * Merges the artifacts of another target. + */ + private Builder addTransitiveDependency(JavaCompilationArgsProvider dep, boolean recursive, + ClasspathType type) { + JavaCompilationArgs args = recursive + ? dep.getRecursiveJavaCompilationArgs() + : dep.getJavaCompilationArgs(); + addTransitiveArgs(args, type); + return this; + } + + /** + * Merges the artifacts of a collection of targets. + */ + public Builder addTransitiveTargets(Iterable<? extends TransitiveInfoCollection> deps) { + return addTransitiveTargets(deps, /*recursive=*/true, ClasspathType.BOTH); + } + + /** + * Includes the contents of another instance of JavaCompilationArgs. + * + * @param args the JavaCompilationArgs instance + * @param type the classpath(s) to consider + */ + public Builder addTransitiveArgs(JavaCompilationArgs args, ClasspathType type) { + if (!ClasspathType.RUNTIME_ONLY.equals(type)) { + compileTimeJarsBuilder.addTransitive(args.getCompileTimeJars()); + } + if (!ClasspathType.COMPILE_ONLY.equals(type)) { + runtimeJarsBuilder.addTransitive(args.getRuntimeJars()); + } + instrumentationMetadataBuilder.addTransitive( + args.getInstrumentationMetadata()); + return this; + } + + /** + * Builds a {@link JavaCompilationArgs} object. + */ + public JavaCompilationArgs build() { + return new JavaCompilationArgs( + runtimeJarsBuilder.build(), + compileTimeJarsBuilder.build(), + instrumentationMetadataBuilder.build()); + } + } + + /** + * Enum to specify transitive compilation args traversal + */ + public static enum ClasspathType { + /* treat the same for compile time and runtime */ + BOTH, + + /* Only include on compile classpath */ + COMPILE_ONLY, + + /* Only include on runtime classpath */ + RUNTIME_ONLY; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgsProvider.java new file mode 100644 index 0000000000..1958ada544 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgsProvider.java @@ -0,0 +1,94 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * An interface for objects that provide information on how to include them in + * Java builds. + */ +@Immutable +public final class JavaCompilationArgsProvider implements TransitiveInfoProvider { + private final JavaCompilationArgs javaCompilationArgs; + private final JavaCompilationArgs recursiveJavaCompilationArgs; + private final NestedSet<Artifact> compileTimeJavaDepArtifacts; + private final NestedSet<Artifact> runTimeJavaDepArtifacts; + + public JavaCompilationArgsProvider(JavaCompilationArgs javaCompilationArgs, + JavaCompilationArgs recursiveJavaCompilationArgs, + NestedSet<Artifact> compileTimeJavaDepArtifacts, + NestedSet<Artifact> runTimeJavaDepArtifacts) { + this.javaCompilationArgs = javaCompilationArgs; + this.recursiveJavaCompilationArgs = recursiveJavaCompilationArgs; + this.compileTimeJavaDepArtifacts = compileTimeJavaDepArtifacts; + this.runTimeJavaDepArtifacts = runTimeJavaDepArtifacts; + } + + public JavaCompilationArgsProvider(JavaCompilationArgs javaCompilationArgs, + JavaCompilationArgs recursiveJavaCompilationArgs) { + this.javaCompilationArgs = javaCompilationArgs; + this.recursiveJavaCompilationArgs = recursiveJavaCompilationArgs; + this.compileTimeJavaDepArtifacts = NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER); + this.runTimeJavaDepArtifacts = NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER); + } + + /** + * Returns non-recursively collected Java compilation information for + * building this target (called when strict_java_deps = 1). + * + * <p>Note that some of the parameters are still collected from the complete + * transitive closure. The non-recursive collection applies mainly to + * compile-time jars. + */ + public JavaCompilationArgs getJavaCompilationArgs() { + return javaCompilationArgs; + } + + /** + * Returns recursively collected Java compilation information for building + * this target (called when strict_java_deps = 0). + */ + public JavaCompilationArgs getRecursiveJavaCompilationArgs() { + return recursiveJavaCompilationArgs; + } + + /** + * Returns non-recursively collected Java dependency artifacts for + * computing a restricted classpath when building this target (called when + * strict_java_deps = 1). + * + * <p>Note that dependency artifacts are needed only when non-recursive + * compilation args do not provide a safe super-set of dependencies. + * Non-strict targets such as proto_library, always collecting their + * transitive closure of deps, do not need to provide dependency artifacts. + */ + public NestedSet<Artifact> getCompileTimeJavaDependencyArtifacts() { + return compileTimeJavaDepArtifacts; + } + + /** + * Returns Java dependency artifacts for computing a restricted run-time + * classpath (called when strict_java_deps = 1). + */ + public NestedSet<Artifact> getRunTimeJavaDependencyArtifacts() { + return runTimeJavaDepArtifacts; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArtifacts.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArtifacts.java new file mode 100644 index 0000000000..98ccbac745 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArtifacts.java @@ -0,0 +1,148 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.LinkedHashSet; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A collection of artifacts for java compilations. It concisely describes the + * outputs of a java-related rule, with runtime jars, compile-time jars, + * unfiltered compile-time jars (these are run through ijar if they are + * dependent upon by another target), source ijars, and instrumentation + * manifests. Not all rules generate all kinds of artifacts. Each java-related + * rule should add both a runtime jar and either a compile-time jar or an + * unfiltered compile-time jar. + * + * <p>An instance of this class only collects the data for the current target, + * not for the transitive closure of targets, so these still need to be + * collected using some other mechanism, such as the {@link + * JavaCompilationArgsProvider}. + */ +@Immutable +public final class JavaCompilationArtifacts { + + public static final JavaCompilationArtifacts EMPTY = new Builder().build(); + + private final ImmutableList<Artifact> runtimeJars; + private final ImmutableList<Artifact> compileTimeJars; + private final ImmutableList<Artifact> instrumentationMetadata; + private final Artifact compileTimeDependencyArtifact; + private final Artifact runTimeDependencyArtifact; + private final Artifact instrumentedJar; + + private JavaCompilationArtifacts(ImmutableList<Artifact> runtimeJars, + ImmutableList<Artifact> compileTimeJars, + ImmutableList<Artifact> instrumentationMetadata, + Artifact compileTimeDependencyArtifact, Artifact runTimeDependencyArtifact, + Artifact instrumentedJar) { + this.runtimeJars = runtimeJars; + this.compileTimeJars = compileTimeJars; + this.instrumentationMetadata = instrumentationMetadata; + this.compileTimeDependencyArtifact = compileTimeDependencyArtifact; + this.runTimeDependencyArtifact = runTimeDependencyArtifact; + this.instrumentedJar = instrumentedJar; + } + + public ImmutableList<Artifact> getRuntimeJars() { + return runtimeJars; + } + + public ImmutableList<Artifact> getCompileTimeJars() { + return compileTimeJars; + } + + public ImmutableList<Artifact> getInstrumentationMetadata() { + return instrumentationMetadata; + } + + public Artifact getCompileTimeDependencyArtifact() { + return compileTimeDependencyArtifact; + } + + public Artifact getRunTimeDependencyArtifact() { + return runTimeDependencyArtifact; + } + + public Artifact getInstrumentedJar() { + return instrumentedJar; + } + + /** + * A builder for {@link JavaCompilationArtifacts}. + */ + public static final class Builder { + private final Set<Artifact> runtimeJars = new LinkedHashSet<>(); + private final Set<Artifact> compileTimeJars = new LinkedHashSet<>(); + private final Set<Artifact> instrumentationMetadata = new LinkedHashSet<>(); + private Artifact compileTimeDependencies; + private Artifact runTimeDependencies; + private Artifact instrumentedJar; + + public JavaCompilationArtifacts build() { + return new JavaCompilationArtifacts(ImmutableList.copyOf(runtimeJars), + ImmutableList.copyOf(compileTimeJars), + ImmutableList.copyOf(instrumentationMetadata), + compileTimeDependencies, runTimeDependencies, instrumentedJar); + } + + public Builder addRuntimeJar(Artifact jar) { + this.runtimeJars.add(jar); + return this; + } + + public Builder addRuntimeJars(Iterable<Artifact> jars) { + Iterables.addAll(this.runtimeJars, jars); + return this; + } + + public Builder addCompileTimeJar(Artifact jar) { + this.compileTimeJars.add(jar); + return this; + } + + public Builder addCompileTimeJars(Iterable<Artifact> jars) { + Iterables.addAll(this.compileTimeJars, jars); + return this; + } + + public Builder addInstrumentationMetadata(Artifact instrumentationMetadata) { + this.instrumentationMetadata.add(instrumentationMetadata); + return this; + } + + public Builder setCompileTimeDependencies(@Nullable Artifact compileTimeDependencies) { + this.compileTimeDependencies = compileTimeDependencies; + return this; + } + + public Builder setRunTimeDependencies(@Nullable Artifact runTimeDependencies) { + this.runTimeDependencies = runTimeDependencies; + return this; + } + + public Builder setInstrumentedJar(@Nullable Artifact instrumentedJar) { + this.instrumentedJar = instrumentedJar; + return this; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java new file mode 100644 index 0000000000..181dd12a61 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java @@ -0,0 +1,436 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import static com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode.OFF; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType; +import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode; +import com.google.devtools.build.lib.vfs.FileSystemUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import javax.annotation.Nullable; + +/** + * A helper class for compiling Java targets. It contains method to create the + * various intermediate Artifacts for using ijars and source ijars. + * <p> + * Also supports the creation of resource and source only Jars. + */ +public class JavaCompilationHelper extends BaseJavaCompilationHelper { + private Artifact outputDepsProtoArtifact; + private JavaTargetAttributes.Builder attributes; + private JavaTargetAttributes builtAttributes; + private final ImmutableList<String> customJavacOpts; + private final List<Artifact> translations = new ArrayList<>(); + private boolean translationsFrozen = false; + private final JavaSemantics semantics; + + public JavaCompilationHelper(RuleContext ruleContext, JavaSemantics semantics, + ImmutableList<String> javacOpts, JavaTargetAttributes.Builder attributes) { + super(ruleContext); + this.attributes = attributes; + this.customJavacOpts = javacOpts; + this.semantics = semantics; + } + + public JavaCompilationHelper(RuleContext ruleContext, JavaSemantics semantics, + JavaTargetAttributes.Builder attributes) { + this(ruleContext, semantics, getDefaultJavacOptsFromRule(ruleContext), attributes); + } + + public JavaTargetAttributes getAttributes() { + if (builtAttributes == null) { + builtAttributes = attributes.build(); + } + return builtAttributes; + } + + /** + * Creates the Action that compiles Java source files. + * + * @param outputJar the class jar Artifact to create with the Action + * @param gensrcOutputJar the generated sources jar Artifact to create with the Action + * (null if no sources will be generated). + * @param outputDepsProto the compiler-generated jdeps file to create with the Action + * (null if not requested) + * @param outputMetadata metadata file (null if no instrumentation is needed). + */ + public void createCompileAction(Artifact outputJar, @Nullable Artifact gensrcOutputJar, + @Nullable Artifact outputDepsProto, @Nullable Artifact outputMetadata) { + JavaTargetAttributes attributes = getAttributes(); + List<String> javacOpts = getJavacOpts(); + JavaCompileAction.Builder builder = createJavaCompileActionBuilder(semantics); + builder.setClasspathEntries(attributes.getCompileTimeClassPath()); + builder.addResources(attributes.getResources()); + builder.addClasspathResources(attributes.getClassPathResources()); + // Only add default bootclasspath entries if not explicitly set in attributes. + if (!attributes.getBootClassPath().isEmpty()) { + builder.setBootclasspathEntries(attributes.getBootClassPath()); + } else { + builder.setBootclasspathEntries(getBootClasspath()); + } + builder.setLangtoolsJar(getLangtoolsJar()); + builder.setJavaBuilderJar(getJavaBuilderJar()); + builder.addTranslations(getTranslations()); + builder.setOutputJar(outputJar); + builder.setGensrcOutputJar(gensrcOutputJar); + builder.setOutputDepsProto(outputDepsProto); + builder.setMetadata(outputMetadata); + builder.addSourceFiles(attributes.getSourceFiles()); + builder.addSourceJars(attributes.getSourceJars()); + builder.setJavacOpts(javacOpts); + builder.setCompressJar(true); + builder.setClassDirectory(outputDir(outputJar)); + builder.setSourceGenDirectory(sourceGenDir(outputJar)); + builder.setTempDirectory(tempDir(outputJar)); + builder.addProcessorPaths(attributes.getProcessorPath()); + builder.addProcessorNames(attributes.getProcessorNames()); + builder.setStrictJavaDeps(attributes.getStrictJavaDeps()); + builder.addDirectJars(attributes.getDirectJars()); + builder.addCompileTimeDependencyArtifacts(attributes.getCompileTimeDependencyArtifacts()); + builder.setRuleKind(attributes.getRuleKind()); + builder.setTargetLabel(attributes.getTargetLabel()); + getAnalysisEnvironment().registerAction(builder.build()); + } + + /** + * Creates the Action that compiles Java source files and optionally instruments them for + * coverage. + * + * @param outputJar the class jar Artifact to create with the Action + * @param gensrcJar the generated sources jar Artifact to create with the Action + * @param outputDepsProto the compiler-generated jdeps file to create with the Action + * @param javaArtifactsBuilder the build to store the instrumentation metadata in + */ + public void createCompileActionWithInstrumentation(Artifact outputJar, Artifact gensrcJar, + Artifact outputDepsProto, JavaCompilationArtifacts.Builder javaArtifactsBuilder) { + createCompileAction(outputJar, gensrcJar, outputDepsProto, + createInstrumentationMetadata(outputJar, javaArtifactsBuilder)); + } + + /** + * Creates the instrumentation metadata artifact if needed. + * + * @return the instrumentation metadata artifact or null if instrumentation is + * disabled + */ + public Artifact createInstrumentationMetadata(Artifact outputJar, + JavaCompilationArtifacts.Builder javaArtifactsBuilder) { + // If we need to instrument the jar, add additional output (the coverage metadata file) to the + // JavaCompileAction. + Artifact instrumentationMetadata = null; + if (shouldInstrumentJar()) { + instrumentationMetadata = semantics.createInstrumentationMetadataArtifact( + getAnalysisEnvironment(), outputJar); + + if (instrumentationMetadata != null) { + javaArtifactsBuilder.addInstrumentationMetadata(instrumentationMetadata); + } + } + return instrumentationMetadata; + } + + private boolean shouldInstrumentJar() { + // TODO(bazel-team): What about source jars? + return getConfiguration().isCodeCoverageEnabled() && attributes.hasSourceFiles() && + getConfiguration().getInstrumentationFilter().isIncluded( + getRuleContext().getLabel().toString()); + } + + /** + * Returns the artifact for a jar file containing source files that were generated by an + * annotation processor or null if no annotation processors are used. + */ + public Artifact createGensrcJar(@Nullable Artifact outputJar) { + if (!usesAnnotationProcessing()) { + return null; + } + return getAnalysisEnvironment().getDerivedArtifact( + FileSystemUtils.appendWithoutExtension(outputJar.getRootRelativePath(), "-gensrc"), + outputJar.getRoot()); + } + + /** + * Returns whether this target uses annotation processing. + */ + private boolean usesAnnotationProcessing() { + JavaTargetAttributes attributes = getAttributes(); + return getJavacOpts().contains("-processor") || !attributes.getProcessorNames().isEmpty(); + } + + public Artifact getOutputDepsProtoArtifact() { + return outputDepsProtoArtifact; + } + /** + * Creates the jdeps file artifact if needed. Returns null if the target can't emit dependency + * information (i.e there is no compilation step, the target acts as an alias). + * + * @param outputJar output jar artifact used to derive the name + * @return the jdeps file artifact or null if the target can't generate such a file + */ + public Artifact createOutputDepsProtoArtifact(Artifact outputJar, + JavaCompilationArtifacts.Builder builder) { + if (!generatesOutputDeps()) { + return null; + } + + outputDepsProtoArtifact = getAnalysisEnvironment().getDerivedArtifact( + FileSystemUtils.replaceExtension(outputJar.getRootRelativePath(), ".jdeps"), + outputJar.getRoot()); + + builder.setRunTimeDependencies(outputDepsProtoArtifact); + return outputDepsProtoArtifact; + } + + /** + * Returns whether this target emits dependency information. Compilation must occur, so certain + * targets acting as aliases have to be filtered out. + */ + private boolean generatesOutputDeps() { + return getJavaConfiguration().getGenerateJavaDeps() && + (attributes.hasSourceFiles() || attributes.hasSourceJars()); + } + + /** + * Creates an Action that packages all of the resources into a Jar. This + * includes the declared resources, the classpath resources and the translated + * messages. + * + * <p>The resource jar artifact is derived from the given original jar, by + * prepending the given prefix and appending the given suffix. The new jar + * uses the same root as the original jar. + */ + // TODO(bazel-team): Extract this method to make it easier to create simple + // zip/jar archives without having to first create a JavaCompilationhelper and + // JavaTargetAttributes. + public Artifact createResourceJarAction(Artifact resourceJar) { + JavaTargetAttributes attributes = getAttributes(); + JavaCompileAction.Builder builder = createJavaCompileActionBuilder(semantics); + builder.setOutputJar(resourceJar); + builder.addResources(attributes.getResources()); + builder.addClasspathResources(attributes.getClassPathResources()); + builder.setLangtoolsJar(getLangtoolsJar()); + builder.addTranslations(getTranslations()); + builder.setCompressJar(true); + builder.setClassDirectory(outputDir(resourceJar)); + builder.setTempDirectory(tempDir(resourceJar)); + builder.setJavaBuilderJar(getJavaBuilderJar()); + getAnalysisEnvironment().registerAction(builder.build()); + return resourceJar; + } + + /** + * Creates an Action that packages the Java source files into a Jar. If {@code gensrcJar} is + * non-null, includes the contents of the {@code gensrcJar} with the output source jar. + * + * @param outputJar the Artifact to create with the Action + * @param gensrcJar the generated sources jar Artifact that should be included with the + * sources in the output Artifact. May be null. + */ + public void createSourceJarAction(Artifact outputJar, @Nullable Artifact gensrcJar) { + JavaTargetAttributes attributes = getAttributes(); + Collection<Artifact> resourceJars = new ArrayList<>(attributes.getSourceJars()); + if (gensrcJar != null) { + resourceJars.add(gensrcJar); + } + createSourceJarAction(semantics, attributes.getSourceFiles(), resourceJars, outputJar); + } + + /** + * Creates the actions that produce the interface jar. Adds the jar artifacts to the given + * JavaCompilationArtifacts builder. + */ + public void createCompileTimeJarAction(Artifact runtimeJar, + @Nullable Artifact runtimeDeps, JavaCompilationArtifacts.Builder builder) { + Artifact jar = getJavaConfiguration().getUseIjars() + ? createIjarAction(runtimeJar, false) + : runtimeJar; + Artifact deps = runtimeDeps; + + builder.addCompileTimeJar(jar); + builder.setCompileTimeDependencies(deps); + } + + /** + * Creates actions that create ijars from generated jars that are an input to + * the Java target. + * + * @return the generated ijars or original jars that are not generated by a + * genrule + */ + public Iterable<Artifact> filterGeneratedJarsThroughIjar(Iterable<Artifact> jars) { + if (!getJavaConfiguration().getUseIjars()) { + return jars; + } + // We need to copy this list in order to avoid generating a new action each time the iterator + // is enumerated + return ImmutableList.copyOf(Iterables.transform(jars, new Function<Artifact, Artifact>() { + @Override + public Artifact apply(Artifact jar) { + return !jar.isSourceArtifact() ? createIjarAction(jar, true) : jar; + } + })); + } + + private void addArgsAndJarsToAttributes(JavaCompilationArgs args, Iterable<Artifact> directJars) { + // Can only be non-null when isStrict() returns true. + if (directJars != null) { + attributes.addDirectCompileTimeClassPathEntries(directJars); + attributes.addDirectJars(directJars); + } + + attributes.merge(args); + } + + private void addLibrariesToAttributesInternal(Iterable<? extends TransitiveInfoCollection> deps) { + JavaCompilationArgs args = JavaCompilationArgs.builder() + .addTransitiveTargets(deps).build(); + + NestedSet<Artifact> directJars = isStrict() + ? getNonRecursiveCompileTimeJarsFromCollection(deps) + : null; + addArgsAndJarsToAttributes(args, directJars); + } + + private void addProvidersToAttributesInternal( + Iterable<? extends SourcesJavaCompilationArgsProvider> deps, boolean isNeverLink) { + JavaCompilationArgs args = JavaCompilationArgs.builder() + .addSourcesTransitiveCompilationArgs(deps, true, + isNeverLink ? ClasspathType.COMPILE_ONLY : ClasspathType.BOTH) + .build(); + + NestedSet<Artifact> directJars = isStrict() + ? getNonRecursiveCompileTimeJarsFromProvider(deps, isNeverLink) + : null; + addArgsAndJarsToAttributes(args, directJars); + } + + private boolean isStrict() { + return getStrictJavaDeps() != OFF; + } + + private NestedSet<Artifact> getNonRecursiveCompileTimeJarsFromCollection( + Iterable<? extends TransitiveInfoCollection> deps) { + JavaCompilationArgs.Builder builder = JavaCompilationArgs.builder(); + builder.addTransitiveTargets(deps, /*recursive=*/false); + return builder.build().getCompileTimeJars(); + } + + private NestedSet<Artifact> getNonRecursiveCompileTimeJarsFromProvider( + Iterable<? extends SourcesJavaCompilationArgsProvider> deps, boolean isNeverLink) { + return JavaCompilationArgs.builder() + .addSourcesTransitiveCompilationArgs(deps, false, + isNeverLink ? ClasspathType.COMPILE_ONLY : ClasspathType.BOTH) + .build().getCompileTimeJars(); + } + + private void addDependencyArtifactsToAttributes( + Iterable<? extends TransitiveInfoCollection> deps) { + NestedSetBuilder<Artifact> compileTimeBuilder = NestedSetBuilder.stableOrder(); + NestedSetBuilder<Artifact> runTimeBuilder = NestedSetBuilder.stableOrder(); + for (JavaCompilationArgsProvider provider : AnalysisUtils.getProviders( + deps, JavaCompilationArgsProvider.class)) { + compileTimeBuilder.addTransitive(provider.getCompileTimeJavaDependencyArtifacts()); + runTimeBuilder.addTransitive(provider.getRunTimeJavaDependencyArtifacts()); + } + attributes.addCompileTimeDependencyArtifacts(compileTimeBuilder.build()); + attributes.addRuntimeDependencyArtifacts(runTimeBuilder.build()); + } + + /** + * Adds the compile time and runtime Java libraries in the transitive closure + * of the deps to the attributes. + * + * @param deps the dependencies to be included as roots of the transitive + * closure + */ + public void addLibrariesToAttributes(Iterable<? extends TransitiveInfoCollection> deps) { + // Enforcing strict Java dependencies: when the --strict_java_deps flag is + // WARN or ERROR, or is DEFAULT and strict_java_deps attribute is unset, + // we use a stricter javac compiler to perform direct deps checks. + attributes.setStrictJavaDeps(getStrictJavaDeps()); + addLibrariesToAttributesInternal(deps); + + JavaClasspathMode classpathMode = getJavaConfiguration().getReduceJavaClasspath(); + if (isStrict() && classpathMode != JavaClasspathMode.OFF) { + addDependencyArtifactsToAttributes(deps); + } + } + + public void addProvidersToAttributes(Iterable<? extends SourcesJavaCompilationArgsProvider> deps, + boolean isNeverLink) { + // see addLibrariesToAttributes() for explanation + attributes.setStrictJavaDeps(getStrictJavaDeps()); + addProvidersToAttributesInternal(deps, isNeverLink); + } + + /** + * Determines whether to enable strict_java_deps. + * + * @return filtered command line flag value, defaulting to ERROR + */ + public StrictDepsMode getStrictJavaDeps() { + return getJavaConfiguration().getFilteredStrictJavaDeps(); + } + + /** + * Gets the value of the "javacopts" attribute combining them with the + * default options. If the current rule has no javacopts attribute, this + * method only returns the default options. + */ + @VisibleForTesting + ImmutableList<String> getJavacOpts() { + return customJavacOpts; + } + + /** + * Obtains the standard list of javac opts needed to build {@code rule}. + * + * This method must only be called during initialization. + * + * @param ruleContext a rule context + * @return a list of options to provide to javac + */ + private static ImmutableList<String> getDefaultJavacOptsFromRule(RuleContext ruleContext) { + return ImmutableList.copyOf(Iterables.concat( + JavaToolchainProvider.getDefaultJavacOptions(ruleContext), + ruleContext.getTokenizedStringListAttr("javacopts"))); + } + + public void addTranslations(Collection<Artifact> translations) { + Preconditions.checkArgument(!translationsFrozen); + this.translations.addAll(translations); + } + + private ImmutableList<Artifact> getTranslations() { + translationsFrozen = true; + return ImmutableList.copyOf(translations); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java new file mode 100644 index 0000000000..655270bcff --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java @@ -0,0 +1,1021 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionInputHelper; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BaseSpawn; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ParameterFile; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.actions.extra.JavaCompileInfo; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine.CustomArgv; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine.CustomMultiArgv; +import com.google.devtools.build.lib.analysis.actions.ParameterFileWriteAction; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.ImmutableIterable; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.util.StringCanonicalizer; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; + +/** + * Action that represents a Java compilation. + */ +@ThreadCompatible +public class JavaCompileAction extends AbstractAction { + + private static final String GUID = "786e174d-ed97-4e79-9f61-ae74430714cf"; + + private static final ResourceSet LOCAL_RESOURCES = + new ResourceSet(750 /*MB*/, 0.5 /*CPU*/, 0.0 /*IO*/); + + private final CommandLine javaCompileCommandLine; + private final CommandLine commandLine; + + /** + * The directory in which generated classfiles are placed. + * May be erased/created by the JavaBuilder. + */ + private final PathFragment classDirectory; + + private final Artifact outputJar; + + /** + * The list of classpath entries to specify to javac. + */ + private final NestedSet<Artifact> classpath; + + /** + * The list of classpath entries to search for annotation processors. + */ + private final ImmutableList<Artifact> processorPath; + + /** + * The list of annotation processor classes to run. + */ + private final ImmutableList<String> processorNames; + + /** + * The translation messages. + */ + private final ImmutableList<Artifact> messages; + + /** + * The set of resources to put into the jar. + */ + private final ImmutableList<Artifact> resources; + + /** + * The set of classpath resources to put into the jar. + */ + private final ImmutableList<Artifact> classpathResources; + + /** + * The set of files which contain lists of additional Java source files to + * compile. + */ + private final ImmutableList<Artifact> sourceJars; + + /** + * The set of explicit Java source files to compile. + */ + private final ImmutableList<Artifact> sourceFiles; + + /** + * The compiler options to pass to javac. + */ + private final ImmutableList<String> javacOpts; + + /** + * The subset of classpath jars provided by direct dependencies. + */ + private final ImmutableList<Artifact> directJars; + + /** + * The level of strict dependency checks (off, warnings, or errors). + */ + private final BuildConfiguration.StrictDepsMode strictJavaDeps; + + /** + * The set of .deps artifacts provided by direct dependencies. + */ + private final ImmutableList<Artifact> compileTimeDependencyArtifacts; + + /** + * The java semantics to get the list of action outputs. + */ + private final JavaSemantics semantics; + + /** + * Constructs an action to compile a set of Java source files to class files. + * + * @param owner the action owner, typically a java_* RuleConfiguredTarget. + * @param baseInputs the set of the input artifacts of the compile action + * without the parameter file action; + * @param outputs the outputs of the action + * @param javaCompileCommandLine the command line for the java library + * builder - it's actually written to the parameter file, but other + * parts (for example, ide_build_info) need access to the data + * @param commandLine the actual invocation command line + */ + private JavaCompileAction(ActionOwner owner, + Iterable<Artifact> baseInputs, + Collection<Artifact> outputs, + CommandLine javaCompileCommandLine, + CommandLine commandLine, + PathFragment classDirectory, + Artifact outputJar, + NestedSet<Artifact> classpath, + List<Artifact> processorPath, + Artifact langtoolsJar, + Artifact javaBuilderJar, + List<String> processorNames, + Collection<Artifact> messages, + Collection<Artifact> resources, + Collection<Artifact> classpathResources, + Collection<Artifact> sourceJars, + Collection<Artifact> sourceFiles, + List<String> javacOpts, + Collection<Artifact> directJars, + BuildConfiguration.StrictDepsMode strictJavaDeps, + Collection<Artifact> compileTimeDependencyArtifacts, + JavaSemantics semantics) { + super(owner, Iterables.concat(ImmutableList.of( + classpath, processorPath, messages, resources, + classpathResources, sourceJars, sourceFiles, compileTimeDependencyArtifacts, + ImmutableList.of(langtoolsJar, javaBuilderJar), baseInputs)), + outputs); + this.javaCompileCommandLine = javaCompileCommandLine; + this.commandLine = commandLine; + + this.classDirectory = Preconditions.checkNotNull(classDirectory); + this.outputJar = outputJar; + this.classpath = classpath; + this.processorPath = ImmutableList.copyOf(processorPath); + this.processorNames = ImmutableList.copyOf(processorNames); + this.messages = ImmutableList.copyOf(messages); + this.resources = ImmutableList.copyOf(resources); + this.classpathResources = ImmutableList.copyOf(classpathResources); + this.sourceJars = ImmutableList.copyOf(sourceJars); + this.sourceFiles = ImmutableList.copyOf(sourceFiles); + this.javacOpts = ImmutableList.copyOf(javacOpts); + this.directJars = ImmutableList.copyOf(directJars); + this.strictJavaDeps = strictJavaDeps; + this.compileTimeDependencyArtifacts = ImmutableList.copyOf(compileTimeDependencyArtifacts); + this.semantics = semantics; + } + + /** + * Returns the given (passed to constructor) source files. + */ + @VisibleForTesting + public Collection<Artifact> getSourceFiles() { + return sourceFiles; + } + + /** + * Returns the list of paths that represent the resources to be added to the + * jar. + */ + @VisibleForTesting + public Collection<Artifact> getResources() { + return resources; + } + + /** + * Returns the list of paths that represents the classpath. + */ + @VisibleForTesting + public Iterable<Artifact> getClasspath() { + return classpath; + } + + /** + * Returns the list of paths that represents the source jars. + */ + @VisibleForTesting + public Collection<Artifact> getSourceJars() { + return sourceJars; + } + + /** + * Returns the list of paths that represents the processor path. + */ + @VisibleForTesting + public List<Artifact> getProcessorpath() { + return processorPath; + } + + @VisibleForTesting + public List<String> getJavacOpts() { + return javacOpts; + } + + @VisibleForTesting + public Collection<Artifact> getDirectJars() { + return directJars; + } + + @VisibleForTesting + public Collection<Artifact> getCompileTimeDependencyArtifacts() { + return compileTimeDependencyArtifacts; + } + + @VisibleForTesting + public BuildConfiguration.StrictDepsMode getStrictJavaDepsMode() { + return strictJavaDeps; + } + + public PathFragment getClassDirectory() { + return classDirectory; + } + + /** + * Returns the list of class names of processors that should + * be run. + */ + @VisibleForTesting + public List<String> getProcessorNames() { + return processorNames; + } + + /** + * Returns the output jar artifact that gets generated by archiving the + * results of the Java compilation and the declared resources. + */ + public Artifact getOutputJar() { + return outputJar; + } + + @Override + public Artifact getPrimaryOutput() { + return getOutputJar(); + } + + /** + * Constructs a command line that can be used to invoke the + * JavaBuilder. + * + * <p>Do not use this method, except for testing (and for the in-process + * strategy). + */ + @VisibleForTesting + public Iterable<String> buildCommandLine() { + return javaCompileCommandLine.arguments(); + } + + /** + * Returns the command and arguments for a java compile action. + */ + public List<String> getCommand() { + return ImmutableList.copyOf(commandLine.arguments()); + } + + @Override + @ThreadCompatible + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + try { + List<ActionInput> outputs = new ArrayList<>(); + outputs.addAll(getOutputs()); + // Add a few useful side-effect output files to the list to retrieve. + // TODO(bazel-team): Just make these Artifacts. + PathFragment classDirectory = getClassDirectory(); + outputs.addAll(semantics.getExtraJavaCompileOutputs(classDirectory)); + outputs.add(ActionInputHelper.fromPath(classDirectory.getChild("srclist").getPathString())); + + try { + // Make sure the directories exist, else the distributor will bomb. + Path classDirectoryPath = executor.getExecRoot().getRelative(getClassDirectory()); + FileSystemUtils.createDirectoryAndParents(classDirectoryPath); + } catch (IOException e) { + throw new EnvironmentalExecException(e.getMessage()); + } + + final ImmutableList<ActionInput> finalOutputs = ImmutableList.copyOf(outputs); + Spawn spawn = new BaseSpawn(getCommand(), ImmutableMap.<String, String>of(), + ImmutableMap.<String, String>of(), this, LOCAL_RESOURCES) { + @Override + public Collection<? extends ActionInput> getOutputFiles() { + return finalOutputs; + } + }; + + executor.getSpawnActionContext(getMnemonic()).exec(spawn, actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException("Java compilation in rule '" + getOwner().getLabel() + "'", + executor.getVerboseFailures(), this); + } + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addStrings(commandLine.arguments()); + return f.hexDigestAndReset(); + } + + @Override + public String describeKey() { + StringBuilder message = new StringBuilder(); + for (String arg : ShellEscaper.escapeAll(commandLine.arguments())) { + message.append(" Command-line argument: "); + message.append(arg); + message.append('\n'); + } + return message.toString(); + } + + @Override + public String getMnemonic() { + return "Javac"; + } + + @Override + protected String getRawProgressMessage() { + int count = sourceFiles.size(); + if (count == 0) { // nothing to compile, just bundling resources and messages + count = resources.size() + classpathResources.size() + messages.size(); + } + return "Building " + outputJar.prettyPrint() + " (" + count + " files)"; + } + + @Override + public String describeStrategy(Executor executor) { + return getContext(executor).strategyLocality(getMnemonic(), true); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + SpawnActionContext context = getContext(executor); + if (context.isRemotable(getMnemonic(), true)) { + return ResourceSet.ZERO; + } + return LOCAL_RESOURCES; + } + + protected SpawnActionContext getContext(Executor executor) { + return executor.getSpawnActionContext(getMnemonic()); + } + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("JavaBuilder "); + Joiner.on(' ').appendTo(result, commandLine.arguments()); + return result.toString(); + } + + @Override + public ExtraActionInfo.Builder getExtraActionInfo() { + JavaCompileInfo.Builder info = JavaCompileInfo.newBuilder(); + info.addAllSourceFile(Artifact.toExecPaths(getSourceFiles())); + info.addAllClasspath(Artifact.toExecPaths(getClasspath())); + info.addClasspath(getClassDirectory().getPathString()); + info.addAllSourcepath(Artifact.toExecPaths(getSourceJars())); + info.addAllJavacOpt(getJavacOpts()); + info.addAllProcessor(getProcessorNames()); + info.addAllProcessorpath(Artifact.toExecPaths(getProcessorpath())); + info.setOutputjar(getOutputJar().getExecPathString()); + + return super.getExtraActionInfo() + .setExtension(JavaCompileInfo.javaCompileInfo, info.build()); + } + + /** + * Creates an instance. + * + * @param configuration the build configuration, which provides the default options and the path + * to the compiler, etc. + * @param classDirectory the directory in which generated classfiles are placed relative to the + * exec root + * @param sourceGenDirectory the directory where source files generated by annotation processors + * should be stored. + * @param tempDirectory a directory in which the library builder can store temporary files + * relative to the exec root + * @param outputJar output jar + * @param compressJar if true compress the output jar + * @param outputDepsProto the proto file capturing dependency information + * @param classpath the complete classpath, the directory in which generated classfiles are placed + * @param processorPath the classpath where javac should search for annotation processors + * @param processorNames the classes that javac should use as annotation processors + * @param messages the message files for translation + * @param resources the set of resources to put into the jar + * @param classpathResources the set of classpath resources to put into the jar + * @param sourceJars the set of jars containing additional source files to compile + * @param sourceFiles the set of explicit Java source files to compile + * @param javacOpts the compiler options to pass to javac + */ + private static CustomCommandLine.Builder javaCompileCommandLine( + final JavaSemantics semantics, + final BuildConfiguration configuration, + final PathFragment classDirectory, + final PathFragment sourceGenDirectory, + PathFragment tempDirectory, + Artifact outputJar, + Artifact gensrcOutputJar, + boolean compressJar, + Artifact outputDepsProto, + final NestedSet<Artifact> classpath, + List<Artifact> processorPath, + Artifact langtoolsJar, + Artifact javaBuilderJar, + List<String> processorNames, + Collection<Artifact> messages, + Collection<Artifact> resources, + Collection<Artifact> classpathResources, + Collection<Artifact> sourceJars, + Collection<Artifact> sourceFiles, + List<String> javacOpts, + final Collection<Artifact> directJars, + BuildConfiguration.StrictDepsMode strictJavaDeps, + Collection<Artifact> compileTimeDependencyArtifacts, + String ruleKind, + Label targetLabel) { + Preconditions.checkNotNull(classDirectory); + Preconditions.checkNotNull(tempDirectory); + Preconditions.checkNotNull(langtoolsJar); + Preconditions.checkNotNull(javaBuilderJar); + + CustomCommandLine.Builder result = CustomCommandLine.builder(); + + result.add("--classdir").addPath(classDirectory); + + result.add("--tempdir").addPath(tempDirectory); + + if (outputJar != null) { + result.addExecPath("--output", outputJar); + } + + if (gensrcOutputJar != null) { + result.add("--sourcegendir").addPath(sourceGenDirectory); + result.addExecPath("--generated_sources_output", gensrcOutputJar); + } + + if (compressJar) { + result.add("--compress_jar"); + } + + if (outputDepsProto != null) { + result.addExecPath("--output_deps_proto", outputDepsProto); + } + + result.add("--classpath").add(new CustomArgv() { + @Override + public String argv() { + List<PathFragment> classpathEntries = new ArrayList<>(); + for (Artifact classpathArtifact : classpath) { + classpathEntries.add(classpathArtifact.getExecPath()); + } + classpathEntries.add(classDirectory); + return Joiner.on(configuration.getHostPathSeparator()).join(classpathEntries); + } + }); + + if (!processorPath.isEmpty()) { + result.addJoinExecPaths("--processorpath", + configuration.getHostPathSeparator(), processorPath); + } + + if (!processorNames.isEmpty()) { + result.add("--processors", processorNames); + } + + if (!messages.isEmpty()) { + result.add("--messages"); + for (Artifact message : messages) { + addAsResourcePrefixedExecPath(semantics, message, result); + } + } + + if (!resources.isEmpty()) { + result.add("--resources"); + for (Artifact resource : resources) { + addAsResourcePrefixedExecPath(semantics, resource, result); + } + } + + if (!classpathResources.isEmpty()) { + result.addExecPaths("--classpath_resources", classpathResources); + } + + if (!sourceJars.isEmpty()) { + result.addExecPaths("--source_jars", sourceJars); + } + + result.addExecPaths("--sources", sourceFiles); + + if (!javacOpts.isEmpty()) { + result.add("--javacopts", javacOpts); + } + + // strict_java_deps controls whether the mapping from jars to targets is + // written out and whether we try to minimize the compile-time classpath. + if (strictJavaDeps != BuildConfiguration.StrictDepsMode.OFF) { + result.add("--strict_java_deps"); + result.add((semantics.useStrictJavaDeps(configuration) ? strictJavaDeps + : BuildConfiguration.StrictDepsMode.OFF).toString()); + result.add(new CustomMultiArgv() { + @Override + public Iterable<String> argv() { + return addJarsToTargets(classpath, directJars); + } + }); + + if (configuration.getFragment(JavaConfiguration.class).getReduceJavaClasspath() + == JavaClasspathMode.JAVABUILDER) { + result.add("--reduce_classpath"); + + if (!compileTimeDependencyArtifacts.isEmpty()) { + result.addExecPaths("--deps_artifacts", compileTimeDependencyArtifacts); + } + } + } + + if (ruleKind != null) { + result.add("--rule_kind"); + result.add(ruleKind); + } + if (targetLabel != null) { + result.add("--target_label"); + if (targetLabel.getPackageIdentifier().getRepository().isDefault()) { + result.add(targetLabel.toString()); + } else { + // @-prefixed strings will be assumed to be filenames and expanded by + // {@link JavaLibraryBuildRequest}, so add an extra &at; to escape it. + result.add("@" + targetLabel); + } + } + + return result; + } + + private static void addAsResourcePrefixedExecPath(JavaSemantics semantics, + Artifact artifact, CustomCommandLine.Builder builder) { + PathFragment execPath = artifact.getExecPath(); + PathFragment resourcePath = semantics.getJavaResourcePath(artifact.getRootRelativePath()); + if (execPath.equals(resourcePath)) { + builder.addPaths(":%s", resourcePath); + } else { + // execPath must end with resourcePath in all cases + PathFragment rootPrefix = trimTail(execPath, resourcePath); + builder.addPaths("%s:%s", rootPrefix, resourcePath); + } + } + + /** + * Returns the root-part of a given path by trimming off the end specified by + * a given tail. Assumes that the tail is known to match, and simply relies on + * the segment lengths. + */ + private static PathFragment trimTail(PathFragment path, PathFragment tail) { + return path.subFragment(0, path.segmentCount() - tail.segmentCount()); + } + + /** + * Builds the list of mappings between jars on the classpath and their + * originating targets names. + */ + private static ImmutableList<String> addJarsToTargets( + NestedSet<Artifact> classpath, Collection<Artifact> directJars) { + ImmutableList.Builder<String> builder = ImmutableList.builder(); + for (Artifact jar : classpath) { + builder.add(directJars.contains(jar) + ? "--direct_dependency" + : "--indirect_dependency"); + builder.add(jar.getExecPathString()); + Label label = getTargetName(jar); + builder.add(label.getPackageIdentifier().getRepository().isDefault() + ? label.toString() + : label.toPathFragment().toString()); + } + return builder.build(); + } + + /** + * Gets the name of the target that produced the given jar artifact. + * + * When specifying jars directly in the "srcs" attribute of a rule (mostly + * for third_party libraries), there is no generating action, so we just + * return the jar name in label form. + */ + private static Label getTargetName(Artifact jar) { + return Preconditions.checkNotNull(jar.getOwner(), jar); + } + + /** + * The actual command line executed for a compile action. + */ + private static CommandLine spawnCommandLine(PathFragment javaExecutable, Artifact javaBuilderJar, + Artifact langtoolsJar, Artifact paramFile, ImmutableList<String> javaBuilderJvmFlags) { + Preconditions.checkNotNull(langtoolsJar); + Preconditions.checkNotNull(javaBuilderJar); + return CustomCommandLine.builder() + .addPath(javaExecutable) + // Langtools jar is placed on the boot classpath so that it can override classes + // in the JRE. Typically this has no effect since langtools.jar does not have + // classes in common with rt.jar. However, it is necessary when using a version + // of javac.jar generated via ant from the langtools build.xml that is of a + // different version than AND has an overlap in contents with the default + // run-time (eg while upgrading the Java version). + .addPaths("-Xbootclasspath/p:%s", langtoolsJar.getExecPath()) + .add(javaBuilderJvmFlags) + .addExecPath("-jar", javaBuilderJar) + .addPaths("@%s", paramFile.getExecPath()) + .build(); + } + + /** + * Builder class to construct Java compile actions. + */ + public static class Builder { + private final ActionOwner owner; + private final AnalysisEnvironment analysisEnvironment; + private final BuildConfiguration configuration; + private final JavaSemantics semantics; + + private PathFragment javaExecutable; + private List<Artifact> javabaseInputs = ImmutableList.of(); + private Artifact outputJar; + private Artifact gensrcOutputJar; + private Artifact outputDepsProto; + private Artifact paramFile; + private Artifact metadata; + private final Collection<Artifact> sourceFiles = new ArrayList<>(); + private final Collection<Artifact> sourceJars = new ArrayList<>(); + private final Collection<Artifact> resources = new ArrayList<>(); + private final Collection<Artifact> classpathResources = new ArrayList<>(); + private final Collection<Artifact> translations = new LinkedHashSet<>(); + private BuildConfiguration.StrictDepsMode strictJavaDeps = + BuildConfiguration.StrictDepsMode.OFF; + private final Collection<Artifact> directJars = new ArrayList<>(); + private final Collection<Artifact> compileTimeDependencyArtifacts = new ArrayList<>(); + private List<String> javacOpts = new ArrayList<>(); + private boolean compressJar; + private NestedSet<Artifact> classpathEntries = + NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER); + private ImmutableList<Artifact> bootclasspathEntries = ImmutableList.of(); + private Artifact javaBuilderJar; + private Artifact langtoolsJar; + private PathFragment classDirectory; + private PathFragment sourceGenDirectory; + private PathFragment tempDirectory; + private final List<Artifact> processorPath = new ArrayList<>(); + private final List<String> processorNames = new ArrayList<>(); + private String ruleKind; + private Label targetLabel; + + /** + * Creates a Builder from an owner and a build configuration. + */ + public Builder(ActionOwner owner, AnalysisEnvironment analysisEnvironment, + BuildConfiguration configuration, JavaSemantics semantics) { + this.owner = owner; + this.analysisEnvironment = analysisEnvironment; + this.configuration = configuration; + this.semantics = semantics; + } + + /** + * Creates a Builder from an owner and a build configuration. + */ + public Builder(RuleContext ruleContext, JavaSemantics semantics) { + this(ruleContext.getActionOwner(), ruleContext.getAnalysisEnvironment(), + ruleContext.getConfiguration(), semantics); + } + + public JavaCompileAction build() { + // TODO(bazel-team): all the params should be calculated before getting here, and the various + // aggregation code below should go away. + List<String> jcopts = new ArrayList<>(javacOpts); + JavaConfiguration javaConfiguration = configuration.getFragment(JavaConfiguration.class); + if (javaConfiguration.getJavaWarns().size() > 0) { + jcopts.add("-Xlint:" + Joiner.on(',').join(javaConfiguration.getJavaWarns())); + } + if (!bootclasspathEntries.isEmpty()) { + jcopts.add("-bootclasspath"); + jcopts.add( + Artifact.joinExecPaths(configuration.getHostPathSeparator(), bootclasspathEntries)); + } + List<String> internedJcopts = new ArrayList<>(); + for (String jcopt : jcopts) { + internedJcopts.add(StringCanonicalizer.intern(jcopt)); + } + + // Invariant: if strictJavaDeps is OFF, then directJars and + // dependencyArtifacts are ignored + if (strictJavaDeps == BuildConfiguration.StrictDepsMode.OFF) { + directJars.clear(); + compileTimeDependencyArtifacts.clear(); + } + + // Invariant: if experimental_java_classpath is not set to 'javabuilder', + // dependencyArtifacts are ignored + if (javaConfiguration.getReduceJavaClasspath() != JavaClasspathMode.JAVABUILDER) { + compileTimeDependencyArtifacts.clear(); + } + + if (paramFile == null) { + paramFile = analysisEnvironment.getDerivedArtifact( + ParameterFile.derivePath(outputJar.getRootRelativePath()), + configuration.getBinDirectory()); + } + + // ImmutableIterable is safe to use here because we know that neither of the components of + // the Iterable.concat() will change. Without ImmutableIterable, AbstractAction will + // waste memory by making a preventive copy of the iterable. + Iterable<Artifact> baseInputs = ImmutableIterable.from(Iterables.concat( + javabaseInputs, + bootclasspathEntries, + ImmutableList.of(paramFile))); + + Preconditions.checkState(javaExecutable != null, owner); + Preconditions.checkState(javaExecutable.isAbsolute() ^ !javabaseInputs.isEmpty(), + javaExecutable); + + Collection<Artifact> outputs; + ImmutableList.Builder<Artifact> outputsBuilder = ImmutableList.builder(); + outputsBuilder.add(outputJar); + if (metadata != null) { + outputsBuilder.add(metadata); + } + if (gensrcOutputJar != null) { + outputsBuilder.add(gensrcOutputJar); + } + if (outputDepsProto != null) { + outputsBuilder.add(outputDepsProto); + } + outputs = outputsBuilder.build(); + + CustomCommandLine.Builder paramFileContentsBuilder = javaCompileCommandLine( + semantics, + configuration, + classDirectory, + sourceGenDirectory, + tempDirectory, + outputJar, + gensrcOutputJar, + compressJar, + outputDepsProto, + classpathEntries, + processorPath, + langtoolsJar, + javaBuilderJar, + processorNames, + translations, + resources, + classpathResources, + sourceJars, + sourceFiles, + internedJcopts, + directJars, + strictJavaDeps, + compileTimeDependencyArtifacts, + ruleKind, + targetLabel); + semantics.buildJavaCommandLine(outputs, configuration, paramFileContentsBuilder); + CommandLine paramFileContents = paramFileContentsBuilder.build(); + Action parameterFileWriteAction = new ParameterFileWriteAction(owner, paramFile, + paramFileContents, ParameterFile.ParameterFileType.UNQUOTED, ISO_8859_1); + analysisEnvironment.registerAction(parameterFileWriteAction); + + CommandLine javaBuilderCommandLine = spawnCommandLine( + javaExecutable, + javaBuilderJar, + langtoolsJar, + paramFile, + javaConfiguration.getDefaultJavaBuilderJvmFlags()); + + return new JavaCompileAction(owner, + baseInputs, + outputs, + paramFileContents, + javaBuilderCommandLine, + classDirectory, + outputJar, + classpathEntries, + processorPath, + langtoolsJar, + javaBuilderJar, + processorNames, + translations, + resources, + classpathResources, + sourceJars, + sourceFiles, + internedJcopts, + directJars, + strictJavaDeps, + compileTimeDependencyArtifacts, + + semantics); + } + + public Builder setParameterFile(Artifact paramFile) { + this.paramFile = paramFile; + return this; + } + + public Builder setJavaExecutable(PathFragment javaExecutable) { + this.javaExecutable = javaExecutable; + return this; + } + + public Builder setJavaBaseInputs(Iterable<Artifact> javabaseInputs) { + this.javabaseInputs = ImmutableList.copyOf(javabaseInputs); + return this; + } + + public Builder setOutputJar(Artifact outputJar) { + this.outputJar = outputJar; + return this; + } + + public Builder setGensrcOutputJar(Artifact gensrcOutputJar) { + this.gensrcOutputJar = gensrcOutputJar; + return this; + } + + public Builder setOutputDepsProto(Artifact outputDepsProto) { + this.outputDepsProto = outputDepsProto; + return this; + } + + public Builder setMetadata(Artifact metadata) { + this.metadata = metadata; + return this; + } + + public Builder addSourceFile(Artifact sourceFile) { + sourceFiles.add(sourceFile); + return this; + } + + public Builder addSourceFiles(Collection<Artifact> sourceFiles) { + this.sourceFiles.addAll(sourceFiles); + return this; + } + + public Builder addSourceJars(Collection<Artifact> sourceJars) { + this.sourceJars.addAll(sourceJars); + return this; + } + + public Builder addResources(Collection<Artifact> resources) { + this.resources.addAll(resources); + return this; + } + + public Builder addClasspathResources(Collection<Artifact> classpathResources) { + this.classpathResources.addAll(classpathResources); + return this; + } + + public Builder addTranslations(Collection<Artifact> translations) { + this.translations.addAll(translations); + return this; + } + + /** + * Sets the strictness of Java dependency checking, see {@link + * com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode}. + */ + public Builder setStrictJavaDeps(BuildConfiguration.StrictDepsMode strictDeps) { + strictJavaDeps = strictDeps; + return this; + } + + /** + * Accumulates the given jar artifacts as being provided by direct dependencies. + */ + public Builder addDirectJars(Collection<Artifact> directJars) { + Iterables.addAll(this.directJars, directJars); + return this; + } + + public Builder addCompileTimeDependencyArtifacts(Collection<Artifact> dependencyArtifacts) { + Iterables.addAll(this.compileTimeDependencyArtifacts, dependencyArtifacts); + return this; + } + + public Builder setJavacOpts(Iterable<String> copts) { + this.javacOpts = ImmutableList.copyOf(copts); + return this; + } + + public Builder setCompressJar(boolean compressJar) { + this.compressJar = compressJar; + return this; + } + + public Builder setClasspathEntries(NestedSet<Artifact> classpathEntries) { + this.classpathEntries = classpathEntries; + return this; + } + + public Builder setBootclasspathEntries(Iterable<Artifact> bootclasspathEntries) { + this.bootclasspathEntries = ImmutableList.copyOf(bootclasspathEntries); + return this; + } + + public Builder setClassDirectory(PathFragment classDirectory) { + this.classDirectory = classDirectory; + return this; + } + + /** + * Sets the directory where source files generated by annotation processors should be stored. + */ + public Builder setSourceGenDirectory(PathFragment sourceGenDirectory) { + this.sourceGenDirectory = sourceGenDirectory; + return this; + } + + public Builder setTempDirectory(PathFragment tempDirectory) { + this.tempDirectory = tempDirectory; + return this; + } + + public Builder addProcessorPaths(Collection<Artifact> processorPaths) { + this.processorPath.addAll(processorPaths); + return this; + } + + public Builder addProcessorNames(Collection<String> processorNames) { + this.processorNames.addAll(processorNames); + return this; + } + + public Builder setLangtoolsJar(Artifact langtoolsJar) { + this.langtoolsJar = langtoolsJar; + return this; + } + + public Builder setJavaBuilderJar(Artifact javaBuilderJar) { + this.javaBuilderJar = javaBuilderJar; + return this; + } + + public Builder setRuleKind(String ruleKind) { + this.ruleKind = ruleKind; + return this; + } + + public Builder setTargetLabel(Label targetLabel) { + this.targetLabel = targetLabel; + return this; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java new file mode 100644 index 0000000000..e1c6dc2b2e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java @@ -0,0 +1,260 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap.Builder; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.common.options.TriState; + +import java.util.List; + +/** + * A java compiler configuration containing the flags required for compilation. + */ +@Immutable +@SkylarkModule(name = "java_configuration", doc = "A java compiler configuration") +public final class JavaConfiguration extends Fragment { + /** + * Values for the --experimental_java_classpath option + */ + public static enum JavaClasspathMode { + /** Use full transitive classpaths, the default behavior. */ + OFF, + /** JavaBuilder computes the reduced classpath before invoking javac. */ + JAVABUILDER, + /** Blaze computes the reduced classpath before invoking JavaBuilder. */ + BLAZE + } + + private final ImmutableList<String> commandLineJavacFlags; + private final Label javaLauncherLabel; + private final Label javaBuilderTop; + private final ImmutableList<String> defaultJavaBuilderJvmOpts; + private final Label javaLangtoolsJar; + private final boolean useIjars; + private final boolean generateJavaDeps; + private final JavaClasspathMode experimentalJavaClasspath; + private final ImmutableList<String> javaWarns; + private final ImmutableList<String> defaultJvmFlags; + private final ImmutableList<String> checkedConstraints; + private final StrictDepsMode strictJavaDeps; + private final Label javacBootclasspath; + private final ImmutableList<String> javacOpts; + private final TriState bundleTranslations; + private final ImmutableList<Label> translationTargets; + private final String javaCpu; + + private final String cacheKey; + private Label javaToolchain; + + JavaConfiguration(boolean generateJavaDeps, + List<String> defaultJvmFlags, JavaOptions javaOptions, Label javaToolchain, String javaCpu, + ImmutableList<String> defaultJavaBuilderJvmOpts) throws InvalidConfigurationException { + this.commandLineJavacFlags = + ImmutableList.copyOf(JavaHelper.tokenizeJavaOptions(javaOptions.javacOpts)); + this.javaLauncherLabel = javaOptions.javaLauncher; + this.javaBuilderTop = javaOptions.javaBuilderTop; + this.defaultJavaBuilderJvmOpts = defaultJavaBuilderJvmOpts; + this.javaLangtoolsJar = javaOptions.javaLangtoolsJar; + this.useIjars = javaOptions.useIjars; + this.generateJavaDeps = generateJavaDeps; + this.experimentalJavaClasspath = javaOptions.experimentalJavaClasspath; + this.javaWarns = ImmutableList.copyOf(javaOptions.javaWarns); + this.defaultJvmFlags = ImmutableList.copyOf(defaultJvmFlags); + this.checkedConstraints = ImmutableList.copyOf(javaOptions.checkedConstraints); + this.strictJavaDeps = javaOptions.strictJavaDeps; + this.javacBootclasspath = javaOptions.javacBootclasspath; + this.javacOpts = ImmutableList.copyOf(javaOptions.javacOpts); + this.bundleTranslations = javaOptions.bundleTranslations; + this.javaCpu = javaCpu; + this.javaToolchain = javaToolchain; + + ImmutableList.Builder<Label> translationsBuilder = ImmutableList.builder(); + for (String s : javaOptions.translationTargets) { + try { + Label label = Label.parseAbsolute(s); + translationsBuilder.add(label); + } catch (SyntaxException e) { + throw new InvalidConfigurationException("Invalid translations target '" + s + "', make " + + "sure it uses correct absolute path syntax.", e); + } + } + this.translationTargets = translationsBuilder.build(); + + this.cacheKey = Joiner.on(" ").join(commandLineJavacFlags); + } + + @SkylarkCallable(name = "default_javac_flags", structField = true, + doc = "The default flags for the Java compiler.") + // TODO(bazel-team): this is the command-line passed options, we should remove from skylark + // probably. + public List<String> getDefaultJavacFlags() { + return commandLineJavacFlags; + } + + @Override + public String cacheKey() { + return cacheKey; + } + + @Override + public void reportInvalidOptions(EventHandler reporter, BuildOptions buildOptions) { + if ((bundleTranslations == TriState.YES) && translationTargets.isEmpty()) { + reporter.handle(Event.error("Translations enabled, but no message translations specified. " + + "Use '--message_translations' to select the message translations to use")); + } + } + + @Override + public void addGlobalMakeVariables(Builder<String, String> globalMakeEnvBuilder) { + globalMakeEnvBuilder.put("JAVA_TRANSLATIONS", buildTranslations() ? "1" : "0"); + globalMakeEnvBuilder.put("JAVA_CPU", javaCpu); + } + + /** + * Returns the Java cpu. + */ + public String getJavaCpu() { + return javaCpu; + } + + /** + * Returns the default javabuilder jar + */ + public Label getDefaultJavaBuilderJar() { + return javaBuilderTop; + } + + /** + * Returns the default JVM flags to be used when invoking javabuilder. + */ + public ImmutableList<String> getDefaultJavaBuilderJvmFlags() { + return defaultJavaBuilderJvmOpts; + } + + /** + * Returns the default java langtools jar + */ + public Label getDefaultJavaLangtoolsJar() { + return javaLangtoolsJar; + } + + /** + * Returns true iff Java compilation should use ijars. + */ + public boolean getUseIjars() { + return useIjars; + } + + /** + * Returns true iff dependency information is generated after compilation. + */ + public boolean getGenerateJavaDeps() { + return generateJavaDeps; + } + + public JavaClasspathMode getReduceJavaClasspath() { + return experimentalJavaClasspath; + } + + /** + * Returns the extra warnings enabled for Java compilation. + */ + public List<String> getJavaWarns() { + return javaWarns; + } + + public List<String> getDefaultJvmFlags() { + return defaultJvmFlags; + } + + public List<String> getCheckedConstraints() { + return checkedConstraints; + } + + public StrictDepsMode getStrictJavaDeps() { + return strictJavaDeps; + } + + public StrictDepsMode getFilteredStrictJavaDeps() { + StrictDepsMode strict = getStrictJavaDeps(); + switch (strict) { + case STRICT: + case DEFAULT: + return StrictDepsMode.ERROR; + default: // OFF, WARN, ERROR + return strict; + } + } + + /** + * @return proper label only if --java_launcher= is specified, otherwise null. + */ + public Label getJavaLauncherLabel() { + return javaLauncherLabel; + } + + public Label getJavacBootclasspath() { + return javacBootclasspath; + } + + public List<String> getJavacOpts() { + return javacOpts; + } + + @Override + public String getName() { + return "Java"; + } + + /** + * Returns the raw translation targets. + */ + public List<Label> getTranslationTargets() { + return translationTargets; + } + + /** + * Returns true if the we should build translations. + */ + public boolean buildTranslations() { + return (bundleTranslations != TriState.NO) && !translationTargets.isEmpty(); + } + + /** + * Returns whether translations were explicitly disabled. + */ + public boolean isTranslationsDisabled() { + return bundleTranslations == TriState.NO; + } + + /** + * Returns the label of the default java_toolchain rule + */ + public Label getToolchainLabel() { + return javaToolchain; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfigurationLoader.java new file mode 100644 index 0000000000..53fdfdfb3b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfigurationLoader.java @@ -0,0 +1,76 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.RedirectChaser; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode; +import com.google.devtools.build.lib.syntax.Label; + +/** + * A loader that creates JavaConfiguration instances based on JavaBuilder configurations and + * command-line options. + */ +public class JavaConfigurationLoader implements ConfigurationFragmentFactory { + private final JavaCpuSupplier cpuSupplier; + + public JavaConfigurationLoader(JavaCpuSupplier cpuSupplier) { + this.cpuSupplier = cpuSupplier; + } + + @Override + public JavaConfiguration create(ConfigurationEnvironment env, BuildOptions buildOptions) + throws InvalidConfigurationException { + JavaOptions javaOptions = buildOptions.get(JavaOptions.class); + + Label javaToolchain = RedirectChaser.followRedirects(env, javaOptions.javaToolchain, + "java_toolchain"); + return create(javaOptions, javaToolchain, cpuSupplier.getJavaCpu(buildOptions, env)); + } + + @Override + public Class<? extends Fragment> creates() { + return JavaConfiguration.class; + } + + public JavaConfiguration create(JavaOptions javaOptions, Label javaToolchain, String javaCpu) + throws InvalidConfigurationException { + + boolean generateJavaDeps = javaOptions.javaDeps || + javaOptions.experimentalJavaClasspath != JavaClasspathMode.OFF; + + ImmutableList<String> defaultJavaBuilderJvmOpts = ImmutableList.<String>builder() + .addAll(getJavacJvmOptions()) + .addAll(JavaHelper.tokenizeJavaOptions(javaOptions.javaBuilderJvmOpts)) + .build(); + + return new JavaConfiguration(generateJavaDeps, javaOptions.jvmOpts, javaOptions, + javaToolchain, javaCpu, defaultJavaBuilderJvmOpts); + } + + /** + * This method returns the list of JVM options when invoking the java compiler. + * + * <p>TODO(bazel-team): Maybe we should put those options in the java_toolchain rule. + */ + protected ImmutableList<String> getJavacJvmOptions() { + return ImmutableList.of("-client"); + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCpuSupplier.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCpuSupplier.java new file mode 100644 index 0000000000..5492abf3d9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCpuSupplier.java @@ -0,0 +1,31 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; + +/** + * Determines the CPU to be used for Java compilation from the build options and the + * configuration environment. + */ +public interface JavaCpuSupplier { + /** + * Returns the Java CPU based on the buiold options and the configuration environment. + */ + String getJavaCpu(BuildOptions buildOptions, ConfigurationEnvironment env) + throws InvalidConfigurationException; +}
\ No newline at end of file diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaExportsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaExportsProvider.java new file mode 100644 index 0000000000..52857f1059 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaExportsProvider.java @@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; + +/** + * The collection of labels of exported targets and artifacts reached via "exports" attribute + * transitively. + */ +@Immutable +public final class JavaExportsProvider implements TransitiveInfoProvider { + + private final NestedSet<Label> transitiveExports; + + public JavaExportsProvider(NestedSet<Label> transitiveExports) { + this.transitiveExports = transitiveExports; + } + + /** + * Returns the labels of exported targets and artifacts reached transitively through the "exports" + * attribute. + */ + public NestedSet<Label> getTransitiveExports() { + return transitiveExports; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaHelper.java new file mode 100644 index 0000000000..b2a7ca0504 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaHelper.java @@ -0,0 +1,104 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.shell.ShellUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * Utility methods for use by Java-related parts of Bazel. + */ +// TODO(bazel-team): Merge with JavaUtil. +public abstract class JavaHelper { + + private JavaHelper() {} + + /** + * Returns the java launcher implementation for the given target, if any. + * A null return value means "use the JDK launcher". + */ + public static TransitiveInfoCollection launcherForTarget(JavaSemantics semantics, + RuleContext ruleContext) { + String launcher = filterLauncherForTarget(semantics, ruleContext); + return (launcher == null) ? null : ruleContext.getPrerequisite(launcher, Mode.TARGET); + } + + /** + * Returns the java launcher artifact for the given target, if any. + * A null return value means "use the JDK launcher". + */ + public static Artifact launcherArtifactForTarget(JavaSemantics semantics, + RuleContext ruleContext) { + String launcher = filterLauncherForTarget(semantics, ruleContext); + return (launcher == null) ? null : ruleContext.getPrerequisiteArtifact(launcher, Mode.TARGET); + } + + /** + * Control structure abstraction for safely extracting a prereq from the launcher attribute + * or --java_launcher flag. + */ + private static String filterLauncherForTarget(JavaSemantics semantics, RuleContext ruleContext) { + // BUILD rule "launcher" attribute + if (ruleContext.getRule().isAttrDefined("launcher", Type.LABEL) + && ruleContext.attributes().get("launcher", Type.LABEL) != null) { + if (ruleContext.attributes().get("launcher", Type.LABEL) + .equals(JavaSemantics.JDK_LAUNCHER_LABEL)) { + return null; + } + return "launcher"; + } + // Blaze flag --java_launcher + JavaConfiguration javaConfig = ruleContext.getFragment(JavaConfiguration.class); + if (ruleContext.getRule().isAttrDefined(":java_launcher", Type.LABEL) + && ((javaConfig.getJavaLauncherLabel() != null + && !javaConfig.getJavaLauncherLabel().equals(JavaSemantics.JDK_LAUNCHER_LABEL)) + || semantics.forceUseJavaLauncherTarget(ruleContext))) { + return ":java_launcher"; + } + return null; + } + + /** + * Javac options require special processing - People use them and expect the + * options to be tokenized. + */ + public static List<String> tokenizeJavaOptions(Iterable<String> inOpts) { + // Ideally, this would be in the options parser. Unfortunately, + // the options parser can't handle a converter that expands + // from a value X into a List<X> and allow-multiple at the + // same time. + List<String> result = new ArrayList<>(); + for (String current : inOpts) { + try { + ShellUtils.tokenize(result, current); + } catch (ShellUtils.TokenizationException ex) { + // Tokenization failed; this likely means that the user + // did not want tokenization to happen on his argument. + // (Any tokenization where we should produce an error + // has already been done by the shell that invoked + // blaze). Therefore, pass the argument through to + // the tool, so that we can see the original error. + result.add(current); + } + } + return result; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaImport.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaImport.java new file mode 100644 index 0000000000..f978f982d5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaImport.java @@ -0,0 +1,201 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TopLevelArtifactProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.cpp.CcLinkParams; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsProvider; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore; +import com.google.devtools.build.lib.rules.cpp.CppCompilationContext; +import com.google.devtools.build.lib.rules.cpp.LinkerInput; +import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType; + +/** + * An implementation for the "java_import" rule. + */ +public class JavaImport implements RuleConfiguredTargetFactory { + private final JavaSemantics semantics; + + protected JavaImport(JavaSemantics semantics) { + this.semantics = semantics; + } + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + ImmutableList<Artifact> srcJars = ImmutableList.of(); + ImmutableList<Artifact> jars = collectJars(ruleContext); + Artifact srcJar = ruleContext.getPrerequisiteArtifact("srcjar", Mode.TARGET); + + if (ruleContext.hasErrors()) { + return null; + } + + ImmutableList<TransitiveInfoCollection> targets = ImmutableList.copyOf( + ruleContext.getPrerequisites("exports", Mode.TARGET)); + final JavaCommon common = new JavaCommon( + ruleContext, semantics, targets, targets, targets); + semantics.checkRule(ruleContext, common); + + // No need for javac options - no compilation happening here. + JavaCompilationHelper helper = new JavaCompilationHelper(ruleContext, semantics, + ImmutableList.<String>of(), new JavaTargetAttributes.Builder(semantics)); + ImmutableMap.Builder<Artifact, Artifact> compilationToRuntimeJarMap = ImmutableMap.builder(); + ImmutableList<Artifact> interfaceJars = + processWithIjar(jars, helper, compilationToRuntimeJarMap); + + common.setJavaCompilationArtifacts(collectJavaArtifacts(jars, interfaceJars)); + + CppCompilationContext transitiveCppDeps = common.collectTransitiveCppDeps(); + NestedSet<LinkerInput> transitiveJavaNativeLibraries = + common.collectTransitiveJavaNativeLibraries(); + + JavaCompilationArgs javaCompilationArgs = common.collectJavaCompilationArgs( + false, common.isNeverLink(), compilationArgsFromSources()); + JavaCompilationArgs recursiveJavaCompilationArgs = common.collectJavaCompilationArgs( + true, common.isNeverLink(), compilationArgsFromSources()); + NestedSet<Artifact> transitiveJavaSourceJars = + collectTransitiveJavaSourceJars(ruleContext, srcJar); + if (srcJar != null) { + srcJars = ImmutableList.of(srcJar); + } + + // The "neverlink" attribute is transitive, so if it is enabled, we don't add any + // runfiles from this target or its dependencies. + Runfiles runfiles = common.isNeverLink() ? + Runfiles.EMPTY : + new Runfiles.Builder() + // add the jars to the runfiles + .addArtifacts(common.getJavaCompilationArtifacts().getRuntimeJars()) + .addTargets(targets, RunfilesProvider.DEFAULT_RUNFILES) + .addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES) + .addTargets(targets, JavaRunfilesProvider.TO_RUNFILES) + .add(ruleContext, JavaRunfilesProvider.TO_RUNFILES) + .build(); + + CcLinkParamsStore ccLinkParamsStore = new CcLinkParamsStore() { + @Override + protected void collect(CcLinkParams.Builder builder, boolean linkingStatically, + boolean linkShared) { + Iterable<? extends TransitiveInfoCollection> deps = + common.targetsTreatedAsDeps(ClasspathType.BOTH); + builder.addTransitiveTargets(deps); + builder.addTransitiveLangTargets(deps, JavaCcLinkParamsProvider.TO_LINK_PARAMS); + } + }; + RuleConfiguredTargetBuilder ruleBuilder = + new RuleConfiguredTargetBuilder(ruleContext); + NestedSetBuilder<Artifact> filesBuilder = NestedSetBuilder.stableOrder(); + filesBuilder.addAll(jars); + + semantics.addProviders( + ruleContext, common, ImmutableList.<String>of(), null, + srcJar, null, compilationToRuntimeJarMap.build(), helper, filesBuilder, ruleBuilder); + + NestedSet<Artifact> filesToBuild = filesBuilder.build(); + + common.addTransitiveInfoProviders(ruleBuilder, filesToBuild, null); + return ruleBuilder + .setFilesToBuild(filesToBuild) + .add(JavaNeverlinkInfoProvider.class, new JavaNeverlinkInfoProvider(common.isNeverLink())) + .add(RunfilesProvider.class, RunfilesProvider.simple(runfiles)) + .add(CcLinkParamsProvider.class, new CcLinkParamsProvider(ccLinkParamsStore)) + .add(JavaCompilationArgsProvider.class, new JavaCompilationArgsProvider( + javaCompilationArgs, recursiveJavaCompilationArgs)) + .add(JavaNativeLibraryProvider.class, new JavaNativeLibraryProvider( + transitiveJavaNativeLibraries)) + .add(CppCompilationContext.class, transitiveCppDeps) + .add(JavaSourceJarsProvider.class, new JavaSourceJarsProvider( + transitiveJavaSourceJars, srcJars)) + .add(TopLevelArtifactProvider.class, new TopLevelArtifactProvider( + JavaSemantics.SOURCE_JARS_OUTPUT_GROUP, transitiveJavaSourceJars)) + .build(); + } + + private NestedSet<Artifact> collectTransitiveJavaSourceJars(RuleContext ruleContext, + Artifact srcJar) { + NestedSetBuilder<Artifact> transitiveJavaSourceJarBuilder = + NestedSetBuilder.stableOrder(); + if (srcJar != null) { + transitiveJavaSourceJarBuilder.add(srcJar); + } + for (JavaSourceJarsProvider other : + ruleContext.getPrerequisites("exports", Mode.TARGET, JavaSourceJarsProvider.class)) { + transitiveJavaSourceJarBuilder.addTransitive(other.getTransitiveSourceJars()); + } + return transitiveJavaSourceJarBuilder.build(); + } + + private JavaCompilationArtifacts collectJavaArtifacts( + ImmutableList<Artifact> jars, + ImmutableList<Artifact> interfaceJars) { + JavaCompilationArtifacts.Builder javaArtifactsBuilder = new JavaCompilationArtifacts.Builder(); + javaArtifactsBuilder.addRuntimeJars(jars); + // interfaceJars Artifacts have proper owner labels + javaArtifactsBuilder.addCompileTimeJars(interfaceJars); + return javaArtifactsBuilder.build(); + } + + private ImmutableList<Artifact> collectJars(RuleContext ruleContext) { + ImmutableList.Builder<Artifact> jarsBuilder = ImmutableList.builder(); + for (TransitiveInfoCollection info : ruleContext.getPrerequisites("jars", Mode.TARGET)) { + if (info.getProvider(JavaCompilationArgsProvider.class) != null) { + ruleContext.attributeError("jars", "should not refer to Java rules"); + } + for (Artifact jar : info.getProvider(FileProvider.class).getFilesToBuild()) { + if (!JavaSemantics.JAR.matches(jar.getFilename())) { + ruleContext.attributeError("jars", jar.getFilename() + " is not a .jar file"); + } else { + jarsBuilder.add(jar); + } + } + } + return jarsBuilder.build(); + } + + private ImmutableList<Artifact> processWithIjar(ImmutableList<Artifact> jars, + JavaCompilationHelper helper, + ImmutableMap.Builder<Artifact, Artifact> compilationToRuntimeJarMap) { + ImmutableList.Builder<Artifact> interfaceJarsBuilder = ImmutableList.builder(); + for (Artifact jar : jars) { + Artifact ijar = helper.createIjarAction(jar, true); + interfaceJarsBuilder.add(ijar); + compilationToRuntimeJarMap.put(ijar, jar); + } + return interfaceJarsBuilder.build(); + } + + private Iterable<SourcesJavaCompilationArgsProvider> compilationArgsFromSources() { + return ImmutableList.of(); + } + + private ImmutableList<String> getJavaConstraints(RuleContext ruleContext) { + return ImmutableList.copyOf(ruleContext.attributes().get("constraints", Type.STRING_LIST)); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaImportBaseRule.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaImportBaseRule.java new file mode 100644 index 0000000000..b153b58af8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaImportBaseRule.java @@ -0,0 +1,91 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; + +/** + * A base rule for building the java_import rule. + */ +@BlazeRule(name = "$java_import_base", + type = RuleClassType.ABSTRACT, + ancestors = { BaseRuleClasses.RuleBase.class }) +public class JavaImportBaseRule implements RuleDefinition { + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + .add(attr(":host_jdk", LABEL) + .cfg(HOST) + .value(JavaSemantics.HOST_JDK)) + /* <!-- #BLAZE_RULE(java_import).ATTRIBUTE(jars) --> + The list of JAR files provided to Java targets that depend on this target. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("jars", LABEL_LIST) + .mandatory() + .nonEmpty() + .allowedFileTypes(JavaSemantics.JAR)) + /* <!-- #BLAZE_RULE(java_import).ATTRIBUTE(srcjar) --> + A JAR file that contains source code for the compiled JAR files. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("srcjar", LABEL) + .allowedFileTypes(JavaSemantics.SOURCE_JAR, JavaSemantics.JAR) + .direct_compile_time_input()) + .removeAttribute("deps") // only exports are allowed; nothing is compiled + /* <!-- #BLAZE_RULE(java_import).ATTRIBUTE(neverlink) --> + Only use this library for compilation and not at runtime. + ${SYNOPSIS} + Useful if the library will be provided by the runtime environment + during execution. Examples of libraries like this are IDE APIs + for IDE plug-ins or <code>tools.jar</code> for anything running on + a standard JDK. + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("neverlink", BOOLEAN).value(false)) + /* <!-- #BLAZE_RULE(java_import).ATTRIBUTE(constraints) --> + Extra constraints imposed on this rule as a Java library. + ${SYNOPSIS} + See <a href="#java_library.constraints">java_library.constraints</a>. + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("constraints", STRING_LIST) + .orderIndependent() + .nonconfigurable("used in Attribute.validityPredicate implementations (loading time)")) + .build(); + } +} +/*<!-- #BLAZE_RULE (NAME = java_import, TYPE = LIBRARY, FAMILY = Java) --> + +${ATTRIBUTE_SIGNATURE} + + <p>This rule allows the use of precompiled JAR files as libraries for + <code><a href="#java_library">java_library</a></code> rules.</p> + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/ diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibrary.java new file mode 100644 index 0000000000..1831ef02ab --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibrary.java @@ -0,0 +1,244 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TopLevelArtifactProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.cpp.CcLinkParams; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsProvider; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore; +import com.google.devtools.build.lib.rules.cpp.CppCompilationContext; +import com.google.devtools.build.lib.rules.cpp.LinkerInput; +import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * Implementation for the java_library rule. + */ +public class JavaLibrary implements RuleConfiguredTargetFactory { + private final JavaSemantics semantics; + + protected JavaLibrary(JavaSemantics semantics) { + this.semantics = semantics; + } + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + JavaCommon common = new JavaCommon(ruleContext, semantics); + RuleConfiguredTargetBuilder builder = init(ruleContext, common); + return builder != null ? builder.build() : null; + } + + public RuleConfiguredTargetBuilder init(RuleContext ruleContext, final JavaCommon common) { + common.initializeJavacOpts(); + JavaTargetAttributes.Builder attributesBuilder = common.initCommon(); + + // Collect the transitive dependencies. + JavaCompilationHelper helper = new JavaCompilationHelper( + ruleContext, semantics, common.getJavacOpts(), attributesBuilder); + helper.addLibrariesToAttributes(common.targetsTreatedAsDeps(ClasspathType.COMPILE_ONLY)); + helper.addProvidersToAttributes(common.compilationArgsFromSources(), common.isNeverLink()); + + if (ruleContext.hasErrors()) { + return null; + } + + semantics.checkRule(ruleContext, common); + + JavaCompilationArtifacts.Builder javaArtifactsBuilder = new JavaCompilationArtifacts.Builder(); + + if (ruleContext.hasErrors()) { + common.setJavaCompilationArtifacts(JavaCompilationArtifacts.EMPTY); + return null; + } + + JavaConfiguration javaConfig = ruleContext.getFragment(JavaConfiguration.class); + NestedSetBuilder<Artifact> filesBuilder = NestedSetBuilder.stableOrder(); + + JavaTargetAttributes attributes = helper.getAttributes(); + if (attributes.hasJarFiles()) { + // This rule is repackaging some source jars as a java library. + Set<Artifact> jarFiles = attributes.getJarFiles(); + javaArtifactsBuilder.addRuntimeJars(jarFiles); + javaArtifactsBuilder.addCompileTimeJars(attributes.getCompileTimeJarFiles()); + + filesBuilder.addAll(jarFiles); + } + if (attributes.hasMessages()) { + helper.addTranslations(semantics.translate(ruleContext, javaConfig, + attributes.getMessages())); + } + + ruleContext.checkSrcsSamePackage(true); + + Artifact jar = null; + + Artifact srcJar = ruleContext.getImplicitOutputArtifact( + JavaSemantics.JAVA_LIBRARY_SOURCE_JAR); + + Artifact classJar = ruleContext.getImplicitOutputArtifact( + JavaSemantics.JAVA_LIBRARY_CLASS_JAR); + + if (attributes.hasSourceFiles() || attributes.hasSourceJars() || attributes.hasResources() + || attributes.hasMessages()) { + // We only want to add a jar to the classpath of a dependent rule if it has content. + javaArtifactsBuilder.addRuntimeJar(classJar); + jar = classJar; + } + + filesBuilder.add(classJar); + + // The gensrcJar is only created if the target uses annotation processing. Otherwise, + // it is null, and the source jar action will not depend on the compile action. + Artifact gensrcJar = helper.createGensrcJar(classJar); + + Artifact outputDepsProto = helper.createOutputDepsProtoArtifact(classJar, javaArtifactsBuilder); + + helper.createCompileActionWithInstrumentation(classJar, gensrcJar, outputDepsProto, + javaArtifactsBuilder); + helper.createSourceJarAction(srcJar, gensrcJar); + + if ((attributes.hasSourceFiles() || attributes.hasSourceJars()) && jar != null) { + helper.createCompileTimeJarAction(jar, outputDepsProto, + javaArtifactsBuilder); + } + + common.setJavaCompilationArtifacts(javaArtifactsBuilder.build()); + common.setClassPathFragment(new ClasspathConfiguredFragment( + common.getJavaCompilationArtifacts(), attributes, common.isNeverLink())); + CppCompilationContext transitiveCppDeps = common.collectTransitiveCppDeps(); + + NestedSet<Artifact> transitiveSourceJars = common.collectTransitiveSourceJars(srcJar); + + // If sources are empty, treat this library as a forwarding node for dependencies. + JavaCompilationArgs javaCompilationArgs = common.collectJavaCompilationArgs( + false, common.isNeverLink(), common.compilationArgsFromSources()); + JavaCompilationArgs recursiveJavaCompilationArgs = common.collectJavaCompilationArgs( + true, common.isNeverLink(), common.compilationArgsFromSources()); + NestedSet<Artifact> compileTimeJavaDepArtifacts = common.collectCompileTimeDependencyArtifacts( + common.getJavaCompilationArtifacts().getCompileTimeDependencyArtifact()); + NestedSet<Artifact> runTimeJavaDepArtifacts = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + NestedSet<LinkerInput> transitiveJavaNativeLibraries = + common.collectTransitiveJavaNativeLibraries(); + + ImmutableList<String> exportedProcessorClasses = ImmutableList.of(); + NestedSet<Artifact> exportedProcessorClasspath = + NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER); + ImmutableList.Builder<String> processorClasses = ImmutableList.builder(); + NestedSetBuilder<Artifact> processorClasspath = NestedSetBuilder.naiveLinkOrder(); + for (JavaPluginInfoProvider provider : Iterables.concat( + common.getPluginInfoProvidersForAttribute("exported_plugins", Mode.HOST), + common.getPluginInfoProvidersForAttribute("exports", Mode.TARGET))) { + processorClasses.addAll(provider.getProcessorClasses()); + processorClasspath.addTransitive(provider.getProcessorClasspath()); + } + exportedProcessorClasses = processorClasses.build(); + exportedProcessorClasspath = processorClasspath.build(); + + CcLinkParamsStore ccLinkParamsStore = new CcLinkParamsStore() { + @Override + protected void collect(CcLinkParams.Builder builder, boolean linkingStatically, + boolean linkShared) { + Iterable<? extends TransitiveInfoCollection> deps = + common.targetsTreatedAsDeps(ClasspathType.BOTH); + builder.addTransitiveTargets(deps); + builder.addTransitiveLangTargets(deps, JavaCcLinkParamsProvider.TO_LINK_PARAMS); + } + }; + + // The "neverlink" attribute is transitive, so we don't add any + // runfiles from this target or its dependencies. + Runfiles runfiles = Runfiles.EMPTY; + if (!common.isNeverLink()) { + Runfiles.Builder runfilesBuilder = new Runfiles.Builder().addArtifacts( + common.getJavaCompilationArtifacts().getRuntimeJars()); + + + runfilesBuilder.addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES); + runfilesBuilder.add(ruleContext, JavaRunfilesProvider.TO_RUNFILES); + + List<TransitiveInfoCollection> depsForRunfiles = new ArrayList<>(); + if (ruleContext.getRule().isAttrDefined("runtime_deps", Type.LABEL_LIST)) { + depsForRunfiles.addAll(ruleContext.getPrerequisites("runtime_deps", Mode.TARGET)); + } + if (ruleContext.getRule().isAttrDefined("exports", Type.LABEL_LIST)) { + depsForRunfiles.addAll(ruleContext.getPrerequisites("exports", Mode.TARGET)); + } + + runfilesBuilder.addTargets(depsForRunfiles, RunfilesProvider.DEFAULT_RUNFILES); + runfilesBuilder.addTargets(depsForRunfiles, JavaRunfilesProvider.TO_RUNFILES); + + TransitiveInfoCollection launcher = JavaHelper.launcherForTarget(semantics, ruleContext); + if (launcher != null) { + runfilesBuilder.addTarget(launcher, RunfilesProvider.DATA_RUNFILES); + } + + semantics.addRunfilesForLibrary(ruleContext, runfilesBuilder); + runfiles = runfilesBuilder.build(); + } + + RuleConfiguredTargetBuilder builder = + new RuleConfiguredTargetBuilder(ruleContext); + + semantics.addProviders( + ruleContext, common, ImmutableList.<String>of(), classJar, srcJar, gensrcJar, + ImmutableMap.<Artifact, Artifact>of(), helper, filesBuilder, builder); + + NestedSet<Artifact> filesToBuild = filesBuilder.build(); + common.addTransitiveInfoProviders(builder, filesToBuild, classJar); + + builder + .add(RunfilesProvider.class, RunfilesProvider.simple(runfiles)) + .setFilesToBuild(filesToBuild) + .add(JavaNeverlinkInfoProvider.class, new JavaNeverlinkInfoProvider(common.isNeverLink())) + .add(CppCompilationContext.class, transitiveCppDeps) + .add(JavaCompilationArgsProvider.class, new JavaCompilationArgsProvider( + javaCompilationArgs, recursiveJavaCompilationArgs, + compileTimeJavaDepArtifacts, runTimeJavaDepArtifacts)) + .add(CcLinkParamsProvider.class, new CcLinkParamsProvider(ccLinkParamsStore)) + .add(JavaNativeLibraryProvider.class, new JavaNativeLibraryProvider( + transitiveJavaNativeLibraries)) + .add(JavaSourceJarsProvider.class, new JavaSourceJarsProvider( + transitiveSourceJars, ImmutableList.of(srcJar))) + .add(TopLevelArtifactProvider.class, new TopLevelArtifactProvider( + JavaSemantics.SOURCE_JARS_OUTPUT_GROUP, transitiveSourceJars)) + // TODO(bazel-team): this should only happen for java_plugin + .add(JavaPluginInfoProvider.class, new JavaPluginInfoProvider( + exportedProcessorClasses, exportedProcessorClasspath)); + + if (ruleContext.hasErrors()) { + return null; + } + + return builder; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibraryHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibraryHelper.java new file mode 100644 index 0000000000..a28d7fd1af --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibraryHelper.java @@ -0,0 +1,382 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import static com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode.OFF; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.UnmodifiableIterator; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.cpp.CcLinkParams.Builder; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore; +import com.google.devtools.build.lib.rules.cpp.CcSpecificLinkParamsProvider; +import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * A class to create Java compile actions in a way that is consistent with java_library. Rules that + * generate source files and emulate java_library on top of that should use this class + * instead of the lower-level API in JavaCompilationHelper. + * + * <p>Rules that want to use this class are required to have an implicit dependency on the + * Java compiler. + */ +public final class JavaLibraryHelper { + /** + * Function for extracting the {@link JavaCompilationArgs} - note that it also handles .jar files. + */ + private static final Function<TransitiveInfoCollection, JavaCompilationArgsProvider> + TO_COMPILATION_ARGS = new Function<TransitiveInfoCollection, JavaCompilationArgsProvider>() { + @Override + public JavaCompilationArgsProvider apply(TransitiveInfoCollection target) { + return forTarget(target); + } + }; + + /** + * Contains the providers as well as the compilation outputs. + */ + public static final class Info { + private final Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers; + private final JavaCompilationArtifacts compilationArtifacts; + + private Info(Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers, + JavaCompilationArtifacts compilationArtifacts) { + this.providers = Collections.unmodifiableMap(providers); + this.compilationArtifacts = compilationArtifacts; + } + + public Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> getProviders() { + return providers; + } + + public JavaCompilationArtifacts getCompilationArtifacts() { + return compilationArtifacts; + } + } + + private final RuleContext ruleContext; + private final BuildConfiguration configuration; + + private Artifact output; + private final List<Artifact> sourceJars = new ArrayList<>(); + /** + * Contains all the dependencies; these are treated as both compile-time and runtime dependencies. + * Some of these may not be complete configured targets; for backwards compatibility with some + * existing code, we sometimes only have pretend dependencies that only have a single {@link + * JavaCompilationArgsProvider}. + */ + private final List<TransitiveInfoCollection> deps = new ArrayList<>(); + private ImmutableList<String> javacOpts = ImmutableList.of(); + + private StrictDepsMode strictDepsMode = StrictDepsMode.OFF; + private JavaClasspathMode classpathMode = JavaClasspathMode.OFF; + private boolean emitProviders = true; + + public JavaLibraryHelper(RuleContext ruleContext) { + this.ruleContext = ruleContext; + this.configuration = ruleContext.getConfiguration(); + this.classpathMode = ruleContext.getFragment(JavaConfiguration.class).getReduceJavaClasspath(); + } + + /** + * Sets the final output jar; if this is not set, then the {@link #build} method throws an {@link + * IllegalStateException}. Note that this class may generate not just the output itself, but also + * a number of additional intermediate files and outputs. + */ + public JavaLibraryHelper setOutput(Artifact output) { + this.output = output; + return this; + } + + /** + * Adds the given source jars. Any .java files in these jars will be compiled. + */ + public JavaLibraryHelper addSourceJars(Iterable<Artifact> sourceJars) { + Iterables.addAll(this.sourceJars, sourceJars); + return this; + } + + /** + * Adds the given source jars. Any .java files in these jars will be compiled. + */ + public JavaLibraryHelper addSourceJars(Artifact... sourceJars) { + return this.addSourceJars(Arrays.asList(sourceJars)); + } + + /** + * Adds the given compilation args as deps. Avoid this method, and prefer {@link #addDeps} + * instead; this method only exists for backward compatibility and may be removed at any time. + */ + public JavaLibraryHelper addProcessedDeps(JavaCompilationArgs... deps) { + for (JavaCompilationArgs dep : deps) { + this.deps.add(toTransitiveInfoCollection(dep)); + } + return this; + } + + private static TransitiveInfoCollection toTransitiveInfoCollection( + final JavaCompilationArgs args) { + return new TransitiveInfoCollection() { + @Override + public <P extends TransitiveInfoProvider> P getProvider(Class<P> provider) { + if (JavaCompilationArgsProvider.class.equals(provider)) { + return provider.cast(new JavaCompilationArgsProvider(args, args)); + } + return null; + } + + @Override + public Label getLabel() { + throw new UnsupportedOperationException(); + } + + @Override + public BuildConfiguration getConfiguration() { + throw new UnsupportedOperationException(); + } + + @Override + public Object get(String providerKey) { + throw new UnsupportedOperationException(); + } + + @Override + public UnmodifiableIterator<TransitiveInfoProvider> iterator() { + throw new UnsupportedOperationException(); + } + }; + } + + /** + * Adds the given targets as deps. These are used as both compile-time and runtime dependencies. + */ + public JavaLibraryHelper addDeps(Iterable<? extends TransitiveInfoCollection> deps) { + for (TransitiveInfoCollection dep : deps) { + Preconditions.checkArgument(dep.getConfiguration() == null + || dep.getConfiguration().equals(configuration)); + this.deps.add(dep); + } + return this; + } + + /** + * Sets the compiler options. + */ + public JavaLibraryHelper setJavacOpts(Iterable<String> javacOpts) { + this.javacOpts = ImmutableList.copyOf(javacOpts); + return this; + } + + /** + * Sets the mode that determines how strictly dependencies are checked. + */ + public JavaLibraryHelper setStrictDepsMode(StrictDepsMode strictDepsMode) { + this.strictDepsMode = strictDepsMode; + return this; + } + + /** + * Disables all providers, i.e., the resulting {@link Info} object will not contain any providers. + * Avoid this method - having this class compute the providers ensures consistency among all + * clients of this code. + */ + public JavaLibraryHelper noProviders() { + this.emitProviders = false; + return this; + } + + /** + * Creates the compile actions and providers. + */ + public Info build(JavaSemantics semantics) { + Preconditions.checkState(output != null, "must have an output file; use setOutput()"); + JavaTargetAttributes.Builder attributes = new JavaTargetAttributes.Builder(semantics); + attributes.addSourceJars(sourceJars); + addDepsToAttributes(attributes); + attributes.setStrictJavaDeps(strictDepsMode); + attributes.setRuleKind(ruleContext.getRule().getRuleClass()); + attributes.setTargetLabel(ruleContext.getLabel()); + + if (isStrict() && classpathMode != JavaClasspathMode.OFF) { + addDependencyArtifactsToAttributes(attributes); + } + + JavaCompilationArtifacts.Builder artifactsBuilder = new JavaCompilationArtifacts.Builder(); + JavaCompilationHelper helper = + new JavaCompilationHelper(ruleContext, semantics, javacOpts, attributes); + Artifact outputDepsProto = helper.createOutputDepsProtoArtifact(output, artifactsBuilder); + helper.createCompileAction(output, null, outputDepsProto, null); + helper.createCompileTimeJarAction(output, outputDepsProto, artifactsBuilder); + artifactsBuilder.addRuntimeJar(output); + JavaCompilationArtifacts compilationArtifacts = artifactsBuilder.build(); + + Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers = + new LinkedHashMap<>(); + if (emitProviders) { + providers.put(JavaCompilationArgsProvider.class, + collectJavaCompilationArgs(compilationArtifacts)); + providers.put(JavaSourceJarsProvider.class, + new JavaSourceJarsProvider(collectTransitiveJavaSourceJars(), sourceJars)); + providers.put(JavaRunfilesProvider.class, collectJavaRunfiles(compilationArtifacts)); + providers.put(JavaCcLinkParamsProvider.class, + new JavaCcLinkParamsProvider(createJavaCcLinkParamsStore())); + } + return new Info(providers, compilationArtifacts); + } + + private void addDepsToAttributes(JavaTargetAttributes.Builder attributes) { + NestedSet<Artifact> directJars = null; + if (isStrict()) { + directJars = getNonRecursiveCompileTimeJarsFromDeps(); + if (directJars != null) { + attributes.addDirectCompileTimeClassPathEntries(directJars); + attributes.addDirectJars(directJars); + } + } + + JavaCompilationArgs args = JavaCompilationArgs.builder() + .addTransitiveDependencies(transformDeps(), true).build(); + attributes.addCompileTimeClassPathEntries(args.getCompileTimeJars()); + attributes.addRuntimeClassPathEntries(args.getRuntimeJars()); + attributes.addInstrumentationMetadataEntries(args.getInstrumentationMetadata()); + } + + private NestedSet<Artifact> getNonRecursiveCompileTimeJarsFromDeps() { + JavaCompilationArgs.Builder builder = JavaCompilationArgs.builder(); + builder.addTransitiveDependencies(transformDeps(), false); + return builder.build().getCompileTimeJars(); + } + + private void addDependencyArtifactsToAttributes(JavaTargetAttributes.Builder attributes) { + NestedSetBuilder<Artifact> compileTimeBuilder = NestedSetBuilder.stableOrder(); + NestedSetBuilder<Artifact> runTimeBuilder = NestedSetBuilder.stableOrder(); + for (JavaCompilationArgsProvider dep : transformDeps()) { + compileTimeBuilder.addTransitive(dep.getCompileTimeJavaDependencyArtifacts()); + runTimeBuilder.addTransitive(dep.getRunTimeJavaDependencyArtifacts()); + } + attributes.addCompileTimeDependencyArtifacts(compileTimeBuilder.build()); + attributes.addRuntimeDependencyArtifacts(runTimeBuilder.build()); + } + + private Iterable<JavaCompilationArgsProvider> transformDeps() { + return Iterables.transform(deps, TO_COMPILATION_ARGS); + } + + private static JavaCompilationArgsProvider forTarget(TransitiveInfoCollection target) { + if (target.getProvider(JavaCompilationArgsProvider.class) != null) { + // If the target has JavaCompilationArgs, we use those. + return target.getProvider(JavaCompilationArgsProvider.class); + } else { + // Otherwise we look for any jar files. It would be good to remove this, and require + // intermediate java_import rules in these cases. + NestedSet<Artifact> filesToBuild = + target.getProvider(FileProvider.class).getFilesToBuild(); + final List<Artifact> jars = new ArrayList<>(); + Iterables.addAll(jars, FileType.filter(filesToBuild, JavaSemantics.JAR)); + JavaCompilationArgs args = JavaCompilationArgs.builder() + .addCompileTimeJars(jars) + .addRuntimeJars(jars) + .build(); + return new JavaCompilationArgsProvider(args, args); + } + } + + private boolean isStrict() { + return strictDepsMode != OFF; + } + + private JavaCompilationArgsProvider collectJavaCompilationArgs( + JavaCompilationArtifacts compilationArtifacts) { + JavaCompilationArgs javaCompilationArgs = + collectJavaCompilationArgs(compilationArtifacts, false); + JavaCompilationArgs recursiveJavaCompilationArgs = + collectJavaCompilationArgs(compilationArtifacts, true); + return new JavaCompilationArgsProvider(javaCompilationArgs, recursiveJavaCompilationArgs); + } + + /** + * Get compilation arguments for java compilation action. + * + * @param recursive a boolean specifying whether to get transitive + * dependencies + * @return java compilation args + */ + private JavaCompilationArgs collectJavaCompilationArgs( + JavaCompilationArtifacts compilationArtifacts, boolean recursive) { + return JavaCompilationArgs.builder() + .merge(compilationArtifacts) + .addTransitiveDependencies(transformDeps(), recursive) + .build(); + } + + private NestedSet<Artifact> collectTransitiveJavaSourceJars() { + NestedSetBuilder<Artifact> transitiveJavaSourceJarBuilder = + NestedSetBuilder.<Artifact>stableOrder(); + transitiveJavaSourceJarBuilder.addAll(sourceJars); + for (JavaSourceJarsProvider other : ruleContext.getPrerequisites( + "deps", Mode.TARGET, JavaSourceJarsProvider.class)) { + transitiveJavaSourceJarBuilder.addTransitive(other.getTransitiveSourceJars()); + } + return transitiveJavaSourceJarBuilder.build(); + } + + private JavaRunfilesProvider collectJavaRunfiles( + JavaCompilationArtifacts javaCompilationArtifacts) { + Runfiles runfiles = new Runfiles.Builder() + // Compiled templates as well, for API. + .addArtifacts(javaCompilationArtifacts.getRuntimeJars()) + .addTargets(deps, JavaRunfilesProvider.TO_RUNFILES) + .build(); + return new JavaRunfilesProvider(runfiles); + } + + private CcLinkParamsStore createJavaCcLinkParamsStore() { + return new CcLinkParamsStore() { + @Override + protected void collect(Builder builder, boolean linkingStatically, boolean linkShared) { + builder.addTransitiveLangTargets( + deps, + JavaCcLinkParamsProvider.TO_LINK_PARAMS); + builder.addTransitiveTargets(deps); + // TODO(bazel-team): This may need to be optional for some clients of this class. + builder.addTransitiveLangTargets( + deps, + CcSpecificLinkParamsProvider.TO_LINK_PARAMS); + } + }; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaNativeLibraryProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaNativeLibraryProvider.java new file mode 100644 index 0000000000..8be42c0d2d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaNativeLibraryProvider.java @@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.cpp.LinkerInput; + +/** + * A target that provides native libraries in the transitive closure of its deps that are needed for + * executing Java code. + */ +@Immutable +public final class JavaNativeLibraryProvider implements TransitiveInfoProvider { + + private final NestedSet<LinkerInput> transitiveJavaNativeLibraries; + + public JavaNativeLibraryProvider( + NestedSet<LinkerInput> transitiveJavaNativeLibraries) { + this.transitiveJavaNativeLibraries = transitiveJavaNativeLibraries; + } + + /** + * Collects native libraries in the transitive closure of its deps that are needed for executing + * Java code. + */ + public NestedSet<LinkerInput> getTransitiveJavaNativeLibraries() { + return transitiveJavaNativeLibraries; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaNeverlinkInfoProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaNeverlinkInfoProvider.java new file mode 100644 index 0000000000..75b36c1706 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaNeverlinkInfoProvider.java @@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A {@link TransitiveInfoProvider} that provides information about whether a Java archive + * is neverlink. + */ +@Immutable +public final class JavaNeverlinkInfoProvider implements TransitiveInfoProvider { + private final boolean isNeverLink; + + public JavaNeverlinkInfoProvider(boolean isNeverLink) { + this.isNeverLink = isNeverLink; + } + + public boolean isNeverlink() { + return isNeverLink; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java new file mode 100644 index 0000000000..f7ef0c714e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java @@ -0,0 +1,350 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.LabelConverter; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsConverter; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode; +import com.google.devtools.build.lib.analysis.config.DefaultsPackage; +import com.google.devtools.build.lib.analysis.config.FragmentOptions; +import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.common.options.Converters.StringSetConverter; +import com.google.devtools.common.options.EnumConverter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.TriState; + +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Command-line options for building Java targets + */ +public class JavaOptions extends FragmentOptions { + // Defaults value for options + static final String DEFAULT_LANGTOOLS_BOOTCLASSPATH = "//tools/jdk:bootclasspath"; + static final String DEFAULT_LANGTOOLS = "//tools/jdk:langtools"; + static final String DEFAULT_JAVABUILDER = "//tools:java/JavaBuilder_deploy.jar"; + static final String DEFAULT_SINGLEJAR = "//tools:java/SingleJar_deploy.jar"; + static final String DEFAULT_JAVABASE = "//tools/jdk:jdk"; + static final String DEFAULT_IJAR = "//tools:java/ijar"; + static final String DEFAULT_TOOLCHAIN = "//tools/jdk:toolchain"; + + /** + * Converter for the --javawarn option. + */ + public static class JavacWarnConverter extends StringSetConverter { + public JavacWarnConverter() { + super("all", + "cast", + "-cast", + "deprecation", + "-deprecation", + "divzero", + "-divzero", + "empty", + "-empty", + "fallthrough", + "-fallthrough", + "finally", + "-finally", + "none", + "options", + "-options", + "overrides", + "-overrides", + "path", + "-path", + "processing", + "-processing", + "rawtypes", + "-rawtypes", + "serial", + "-serial", + "unchecked", + "-unchecked" + ); + } + } + + /** + * Converter for the --experimental_java_classpath option. + */ + public static class JavaClasspathModeConverter extends EnumConverter<JavaClasspathMode> { + public JavaClasspathModeConverter() { + super(JavaClasspathMode.class, "Java classpath reduction strategy"); + } + } + + @Option(name = "javabase", + defaultValue = DEFAULT_JAVABASE, + category = "version", + help = "JAVABASE used for the JDK invoked by Blaze. This is the " + + "JAVABASE which will be used to execute external Java " + + "commands.") + public String javaBase; + + @Option(name = "java_toolchain", + defaultValue = DEFAULT_TOOLCHAIN, + category = "version", + converter = LabelConverter.class, + help = "The name of the toolchain rule for Java. Default is " + DEFAULT_TOOLCHAIN) + public Label javaToolchain; + + @Option(name = "host_javabase", + defaultValue = DEFAULT_JAVABASE, + category = "version", + help = "JAVABASE used for the host JDK. This is the JAVABASE which is used to execute " + + " tools during a build.") + public String hostJavaBase; + + @Option(name = "javacopt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to javac.") + public List<String> javacOpts; + + @Option(name = "jvmopt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to the Java VM. These options will get added to the " + + "VM startup options of each java_binary target.") + public List<String> jvmOpts; + + @Option(name = "javawarn", + converter = JavacWarnConverter.class, + defaultValue = "", + category = "flags", + allowMultiple = true, + help = "Additional javac warnings to enable when compiling Java source files.") + public List<String> javaWarns; + + @Option(name = "use_ijars", + defaultValue = "true", + category = "strategy", + help = "If enabled, this option causes Java compilation to use interface jars. " + + "This will result in faster incremental compilation, " + + "but error messages can be different.") + public boolean useIjars; + + @Deprecated + @Option(name = "use_src_ijars", + defaultValue = "false", + category = "undocumented", + help = "No-op. Kept here for backwards compatibility.") + public boolean useSourceIjars; + + @Deprecated + @Option(name = "experimental_incremental_ijars", + defaultValue = "false", + category = "undocumented", + help = "No-op. Kept here for backwards compatibility.") + public boolean incrementalIjars; + + @Option(name = "java_deps", + defaultValue = "true", + category = "strategy", + help = "Generate dependency information (for now, compile-time classpath) per Java target.") + public boolean javaDeps; + + @Option(name = "experimental_java_deps", + defaultValue = "false", + category = "experimental", + expansion = "--java_deps", + deprecationWarning = "Use --java_deps instead") + public boolean experimentalJavaDeps; + + @Option(name = "experimental_java_classpath", + allowMultiple = false, + defaultValue = "javabuilder", + converter = JavaClasspathModeConverter.class, + category = "semantics", + help = "Enables reduced classpaths for Java compilations.") + public JavaClasspathMode experimentalJavaClasspath; + + @Option(name = "java_cpu", + defaultValue = "null", + category = "semantics", + help = "The Java target CPU. Default is k8.") + public String javaCpu; + + @Option(name = "java_debug", + defaultValue = "null", + category = "testing", + expansion = {"--test_arg=--wrapper_script_flag=--debug", "--test_output=streamed", + "--test_strategy=exclusive", "--test_timeout=9999", "--nocache_test_results"}, + help = "Causes the Java virtual machine of a java test to wait for a connection from a " + + "JDWP-compliant debugger (such as jdb) before starting the test. Implies " + + "-test_output=streamed." + ) + public Void javaTestDebug; + + @Option(name = "strict_java_deps", + allowMultiple = false, + defaultValue = "default", + converter = StrictDepsConverter.class, + category = "semantics", + help = "If true, checks that a Java target explicitly declares all directly used " + + "targets as dependencies.") + public StrictDepsMode strictJavaDeps; + + @Option(name = "javabuilder_top", + defaultValue = DEFAULT_JAVABUILDER, + category = "version", + converter = LabelConverter.class, + help = "Label of the filegroup that contains the JavaBuilder jar.") + public Label javaBuilderTop; + + @Option(name = "javabuilder_jvmopt", + allowMultiple = true, + defaultValue = "", + category = "undocumented", + help = "Additional options to pass to the JVM when invoking JavaBuilder.") + public List<String> javaBuilderJvmOpts; + + @Option(name = "singlejar_top", + defaultValue = DEFAULT_SINGLEJAR, + category = "version", + converter = LabelConverter.class, + help = "Label of the filegroup that contains the SingleJar jar.") + public Label singleJarTop; + + @Option(name = "ijar_top", + defaultValue = DEFAULT_IJAR, + category = "version", + converter = LabelConverter.class, + help = "Label of the filegroup that contains the ijar binary.") + public Label iJarTop; + + @Option(name = "java_langtools", + defaultValue = DEFAULT_LANGTOOLS, + category = "version", + converter = LabelConverter.class, + help = "Label of the rule that produces the Java langtools jar.") + public Label javaLangtoolsJar; + + @Option(name = "javac_bootclasspath", + defaultValue = DEFAULT_LANGTOOLS_BOOTCLASSPATH, + category = "version", + converter = LabelConverter.class, + help = "Label of the rule that produces the bootclasspath jars for javac to use.") + public Label javacBootclasspath; + + @Option(name = "java_launcher", + defaultValue = "null", + converter = LabelConverter.class, + category = "semantics", + help = "If enabled, a specific Java launcher is used. " + + "The \"launcher\" attribute overrides this flag. ") + public Label javaLauncher; + + @Option(name = "translations", + defaultValue = "auto", + category = "semantics", + help = "Translate Java messages; bundle all translations into the jar " + + "for each affected rule.") + public TriState bundleTranslations; + + @Option(name = "message_translations", + defaultValue = "", + category = "semantics", + allowMultiple = true, + help = "The message translations used for translating messages in Java targets.") + public List<String> translationTargets; + + @Option(name = "check_constraint", + allowMultiple = true, + defaultValue = "", + category = "checking", + help = "Check the listed constraint.") + public List<String> checkedConstraints; + + @Override + public FragmentOptions getHost(boolean fallback) { + JavaOptions host = (JavaOptions) getDefault(); + + host.javaBase = hostJavaBase; + host.jvmOpts = ImmutableList.of("-client", "-XX:ErrorFile=/dev/stderr"); + + host.javacOpts = javacOpts; + host.javaLangtoolsJar = javaLangtoolsJar; + host.javaBuilderTop = javaBuilderTop; + host.javaToolchain = javaToolchain; + host.singleJarTop = singleJarTop; + host.iJarTop = iJarTop; + + // Java builds often contain complicated code generators for which + // incremental build performance is important. + host.useIjars = useIjars; + + host.javaDeps = javaDeps; + host.experimentalJavaClasspath = experimentalJavaClasspath; + + return host; + } + + @Override + public void addAllLabels(Multimap<String, Label> labelMap) { + addOptionalLabel(labelMap, "jdk", javaBase); + addOptionalLabel(labelMap, "jdk", hostJavaBase); + if (javaLauncher != null) { + labelMap.put("java_launcher", javaLauncher); + } + labelMap.put("javabuilder", javaBuilderTop); + labelMap.put("singlejar", singleJarTop); + labelMap.put("ijar", iJarTop); + labelMap.put("java_toolchain", javaToolchain); + labelMap.putAll("translation", getTranslationLabels()); + } + + @Override + public Map<String, Set<Label>> getDefaultsLabels(BuildConfiguration.Options commonOptions) { + Set<Label> jdkLabels = new LinkedHashSet<>(); + DefaultsPackage.parseAndAdd(jdkLabels, javaBase); + DefaultsPackage.parseAndAdd(jdkLabels, hostJavaBase); + Map<String, Set<Label>> result = new HashMap<>(); + result.put("JDK", jdkLabels); + result.put("JAVA_LANGTOOLS", ImmutableSet.of(javaLangtoolsJar)); + result.put("JAVAC_BOOTCLASSPATH", ImmutableSet.of(javacBootclasspath)); + result.put("JAVABUILDER", ImmutableSet.of(javaBuilderTop)); + result.put("SINGLEJAR", ImmutableSet.of(singleJarTop)); + result.put("IJAR", ImmutableSet.of(iJarTop)); + result.put("JAVA_TOOLCHAIN", ImmutableSet.of(javaToolchain)); + + return result; + } + + private Set<Label> getTranslationLabels() { + Set<Label> result = new LinkedHashSet<>(); + for (String s : translationTargets) { + try { + Label label = Label.parseAbsolute(s); + result.add(label); + } catch (SyntaxException e) { + // We ignore this exception here - it will cause an error message at a later time. + } + } + return result; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPlugin.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPlugin.java new file mode 100644 index 0000000000..526d52c01d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPlugin.java @@ -0,0 +1,57 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * Implementation for the java_plugin rule. + */ +public class JavaPlugin implements RuleConfiguredTargetFactory { + + private final JavaSemantics semantics; + + protected JavaPlugin(JavaSemantics semantics) { + this.semantics = semantics; + } + + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + JavaLibrary javaLibrary = new JavaLibrary(semantics); + JavaCommon common = new JavaCommon(ruleContext, semantics); + RuleConfiguredTargetBuilder builder = javaLibrary.init(ruleContext, common); + if (builder == null) { + return null; + } + builder.add(JavaPluginInfoProvider.class, new JavaPluginInfoProvider( + getProcessorClasses(ruleContext), common.getRuntimeClasspath())); + return builder.build(); + } + + /** + * Returns the class that should be passed to javac in order + * to run the annotation processor this class represents. + */ + private ImmutableList<String> getProcessorClasses(RuleContext ruleContext) { + if (ruleContext.getRule().isAttributeValueExplicitlySpecified("processor_class")) { + return ImmutableList.of(ruleContext.attributes().get("processor_class", Type.STRING)); + } + return ImmutableList.of(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPluginInfoProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPluginInfoProvider.java new file mode 100644 index 0000000000..520a228966 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPluginInfoProvider.java @@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Provider for users of Java plugins. + */ +@Immutable +public final class JavaPluginInfoProvider implements TransitiveInfoProvider { + + private final ImmutableList<String> processorClasses; + private final NestedSet<Artifact> processorClasspath; + + public JavaPluginInfoProvider(ImmutableList<String> processorClasses, + NestedSet<Artifact> processorClasspath) { + this.processorClasses = processorClasses; + this.processorClasspath = processorClasspath; + } + + /** + * Returns the class that should be passed to javac in order + * to run the annotation processor this class represents. + */ + public ImmutableList<String> getProcessorClasses() { + return processorClasses; + } + + /** + * Returns the artifacts to add to the runtime classpath for this plugin. + */ + public NestedSet<Artifact> getProcessorClasspath() { + return processorClasspath; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPrimaryClassProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPrimaryClassProvider.java new file mode 100644 index 0000000000..fd900111b7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPrimaryClassProvider.java @@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Provides the fully qualified name of the primary class to invoke for java targets. + */ +@Immutable +public final class JavaPrimaryClassProvider implements TransitiveInfoProvider { + + private final String primaryClass; + + public JavaPrimaryClassProvider(String primaryClass) { + this.primaryClass = primaryClass; + } + + /** + * Returns either the Java class whose main() method is to be invoked (when + * use_testrunner=0) or the Java subclass of junit.framework.Test that + * is to be tested by the test runner class (when use_testrunner=1). + * + * @return a fully qualified Java class name, or null if none could be + * determined. + */ + public String getPrimaryClass() { + return primaryClass; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaRunfilesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaRunfilesProvider.java new file mode 100644 index 0000000000..b742d6224b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaRunfilesProvider.java @@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A {@link TransitiveInfoProvider} that supplies runfiles for Java dependencies. + */ +@Immutable +public final class JavaRunfilesProvider implements TransitiveInfoProvider { + private final Runfiles runfiles; + + public JavaRunfilesProvider(Runfiles runfiles) { + this.runfiles = runfiles; + } + + public Runfiles getRunfiles() { + return runfiles; + } + + /** + * Returns a function that gets the Java runfiles from a {@link TransitiveInfoCollection} or + * the empty runfiles instance if it does not contain that provider. + */ + public static final Function<TransitiveInfoCollection, Runfiles> TO_RUNFILES = + new Function<TransitiveInfoCollection, Runfiles>() { + @Override + public Runfiles apply(TransitiveInfoCollection input) { + JavaRunfilesProvider provider = input.getProvider(JavaRunfilesProvider.class); + return provider == null + ? Runfiles.EMPTY + : provider.getRunfiles(); + } + }; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaRuntimeClasspathProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaRuntimeClasspathProvider.java new file mode 100644 index 0000000000..c8090df9a2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaRuntimeClasspathProvider.java @@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Provider for the runtime classpath contributions of a Java binary. + * + * Used to exclude already-available artifacts from related binaries + * (e.g. plugins). + */ +@Immutable +public final class JavaRuntimeClasspathProvider implements TransitiveInfoProvider { + + private final NestedSet<Artifact> runtimeClasspath; + + public JavaRuntimeClasspathProvider(NestedSet<Artifact> runtimeClasspath) { + this.runtimeClasspath = runtimeClasspath; + } + + /** + * Returns the artifacts included on the runtime classpath of this binary. + */ + public NestedSet<Artifact> getRuntimeClasspath() { + return runtimeClasspath; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaSemantics.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaSemantics.java new file mode 100644 index 0000000000..64b62145de --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaSemantics.java @@ -0,0 +1,351 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.LanguageDependentFragment.LibraryLanguage; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.Runfiles.Builder; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Attribute.LateBoundLabel; +import com.google.devtools.build.lib.packages.Attribute.LateBoundLabelList; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.rules.java.DeployArchiveBuilder.Compression; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.List; + +import javax.annotation.Nullable; + +/** + * Pluggable Java compilation semantics. + */ +public interface JavaSemantics { + + public static final LibraryLanguage LANGUAGE = new LibraryLanguage("Java"); + + public static final SafeImplicitOutputsFunction JAVA_LIBRARY_CLASS_JAR = + fromTemplates("lib%{name}.jar"); + public static final SafeImplicitOutputsFunction JAVA_LIBRARY_SOURCE_JAR = + fromTemplates("lib%{name}-src.jar"); + public static final SafeImplicitOutputsFunction JAVA_BINARY_CLASS_JAR = + fromTemplates("%{name}.jar"); + public static final SafeImplicitOutputsFunction JAVA_BINARY_SOURCE_JAR = + fromTemplates("%{name}-src.jar"); + public static final SafeImplicitOutputsFunction JAVA_BINARY_DEPLOY_JAR = + fromTemplates("%{name}_deploy.jar"); + public static final SafeImplicitOutputsFunction JAVA_BINARY_DEPLOY_SOURCE_JAR = + fromTemplates("%{name}_deploy-src.jar"); + + public static final FileType JAVA_SOURCE = FileType.of(".java"); + public static final FileType JAR = FileType.of(".jar"); + public static final FileType PROPERTIES = FileType.of(".properties"); + public static final FileType SOURCE_JAR = FileType.of(".srcjar"); + // TODO(bazel-team): Rename this metadata extension to something meaningful. + public static final FileType COVERAGE_METADATA = FileType.of(".em"); + + /** + * Label to the Java Toolchain rule. It is resolved from a label given in the java options. + */ + static final String JAVA_TOOLCHAIN_LABEL = "//tools/defaults:java_toolchain"; + + public static final LateBoundLabel<BuildConfiguration> JAVA_TOOLCHAIN = + new LateBoundLabel<BuildConfiguration>(JAVA_TOOLCHAIN_LABEL) { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.getFragment(JavaConfiguration.class).getToolchainLabel(); + } + }; + + /** + * Name of the output group used for source jars. + */ + public static final String SOURCE_JARS_OUTPUT_GROUP = "source_jars"; + + /** + * Label of a pseudo-filegroup that contains all jdk files for all + * configurations, as specified on the command-line. + */ + public static final String JDK_LABEL = "//tools/defaults:jdk"; + + /** + * Label of a pseudo-filegroup that contains the boot-classpath entries. + */ + public static final String JAVAC_BOOTCLASSPATH_LABEL = "//tools/defaults:javac_bootclasspath"; + + /** + * Label of the JavaBuilder JAR used for compiling Java source code. + */ + public static final String JAVABUILDER_LABEL = "//tools/defaults:javabuilder"; + + /** + * Label of the SingleJar JAR used for creating deploy jars. + */ + public static final String SINGLEJAR_LABEL = "//tools/defaults:singlejar"; + + /** + * Label of pseudo-cc_binary that tells Blaze a java target's JAVABIN is never to be replaced by + * the contents of --java_launcher; only the JDK's launcher will ever be used. + */ + public static final Label JDK_LAUNCHER_LABEL = + Label.parseAbsoluteUnchecked("//third_party/java/jdk:jdk_launcher"); + + /** + * Implementation for the :jvm attribute. + */ + public static final LateBoundLabel<BuildConfiguration> JVM = + new LateBoundLabel<BuildConfiguration>(JDK_LABEL) { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.getFragment(Jvm.class).getJvmLabel(); + } + }; + + /** + * Implementation for the :host_jdk attribute. + */ + public static final LateBoundLabel<BuildConfiguration> HOST_JDK = + new LateBoundLabel<BuildConfiguration>(JDK_LABEL) { + @Override + public boolean useHostConfiguration() { + return true; + } + + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.getFragment(Jvm.class).getJvmLabel(); + } + }; + + /** + * Implementation for the :java_launcher attribute. Note that the Java launcher is disabled by + * default, so it returns null for the configuration-independent default value. + */ + public static final LateBoundLabel<BuildConfiguration> JAVA_LAUNCHER = + new LateBoundLabel<BuildConfiguration>() { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.getFragment(JavaConfiguration.class).getJavaLauncherLabel(); + } + }; + + public static final LateBoundLabelList<BuildConfiguration> JAVA_PLUGINS = + new LateBoundLabelList<BuildConfiguration>() { + @Override + public List<Label> getDefault(Rule rule, BuildConfiguration configuration) { + return ImmutableList.copyOf(configuration.getPlugins()); + } + }; + + public static final String IJAR_LABEL = "//tools/defaults:ijar"; + + /** + * Verifies if the rule contains and errors. + * + * <p>Errors should be signaled through {@link RuleContext}. + */ + void checkRule(RuleContext ruleContext, JavaCommon javaCommon); + + /** + * Returns the main class of a Java binary. + */ + String getMainClass(RuleContext ruleContext, JavaCommon javaCommon); + + /** + * Returns the resources contributed by a Java rule (usually the contents of the + * {@code resources} attribute) + */ + ImmutableList<Artifact> collectResources(RuleContext ruleContext); + + /** + * Creates the instrumentation metadata artifact for the specified output .jar . + */ + @Nullable Artifact createInstrumentationMetadataArtifact( + AnalysisEnvironment analysisEnvironment, Artifact outputJar); + + /** + * May add extra command line options to the Java compile command line. + */ + void buildJavaCommandLine(Collection<Artifact> outputs, BuildConfiguration configuration, + CustomCommandLine.Builder result); + + + /** + * Constructs the command line to call SingleJar to join all artifacts from + * {@code classpath} (java code) and {@code resources} into {@code output}. + */ + CustomCommandLine buildSingleJarCommandLine(BuildConfiguration configuration, + Artifact output, String mainClass, ImmutableList<String> manifestLines, + Iterable<Artifact> buildInfoFiles, ImmutableList<Artifact> resources, + Iterable<Artifact> classpath, boolean includeBuildData, + Compression compression, Artifact launcher); + + /** + * Creates the action that writes the Java executable stub script. + */ + void createStubAction(RuleContext ruleContext, final JavaCommon javaCommon, + List<String> jvmFlags, Artifact executable, String javaStartClass, + String javaExecutable); + + /** + * Adds extra runfiles for a {@code java_binary} rule. + */ + void addRunfilesForBinary(RuleContext ruleContext, Artifact launcher, + Runfiles.Builder runfilesBuilder); + + /** + * Adds extra runfiles for a {@code java_library} rule. + */ + void addRunfilesForLibrary(RuleContext ruleContext, Runfiles.Builder runfilesBuilder); + + /** + * Returns the coverage instrumentation specification to be used in Java rules. + */ + InstrumentationSpec getCoverageInstrumentationSpec(); + + /** + * Returns the additional options to be passed to javac. + */ + Iterable<String> getExtraJavacOpts(RuleContext ruleContext); + + /** + * Add additional targets to be treated as direct dependencies. + */ + void collectTargetsTreatedAsDeps( + RuleContext ruleContext, ImmutableList.Builder<TransitiveInfoCollection> builder); + + /** + * Enables coverage support for the java target - adds instrumented jar to the classpath and + * modifies main class. + * + * @return new main class + */ + String addCoverageSupport(JavaCompilationHelper helper, + JavaTargetAttributes.Builder attributes, + Artifact executable, Artifact instrumentationMetadata, + JavaCompilationArtifacts.Builder javaArtifactsBuilder, String mainClass); + + /** + * Return the JVM flags to be used in a Java binary. + */ + Iterable<String> getJvmFlags(RuleContext ruleContext, JavaCommon javaCommon, + Artifact launcher, List<String> userJvmFlags); + + /** + * Adds extra providers to a Java target. + */ + void addProviders(RuleContext ruleContext, + JavaCommon javaCommon, + List<String> jvmFlags, + Artifact classJar, + Artifact srcJar, + Artifact gensrcJar, + ImmutableMap<Artifact, Artifact> compilationToRuntimeJarMap, + JavaCompilationHelper helper, + NestedSetBuilder<Artifact> filesBuilder, + RuleConfiguredTargetBuilder ruleBuilder); + + /** + * Tell if a build with the given configuration should use strict java dependencies. This method + * enforces strict java dependencies off if it returns false. + */ + boolean useStrictJavaDeps(BuildConfiguration configuration); + + /** + * Translates XMB messages to translations artifact suitable for Java targets. + */ + Collection<Artifact> translate(RuleContext ruleContext, JavaConfiguration javaConfig, + List<Artifact> messages); + + /** + * Get the launcher artifact for a java binary, creating the necessary actions for it. + * + * @param ruleContext The rule context + * @param common The common helper class. + * @param deployArchiveBuilder the builder to construct the deploy archive action (mutable). + * @param runfilesBuilder the builder to construct the list of runfiles (mutable). + * @param jvmFlags the list of flags to pass to the JVM when running the Java binary (mutable). + * @param attributesBuilder the builder to construct the list of attributes of this target + * (mutable). + * @return the launcher as an artifact + */ + Artifact getLauncher(final RuleContext ruleContext, final JavaCommon common, + DeployArchiveBuilder deployArchiveBuilder, Runfiles.Builder runfilesBuilder, + List<String> jvmFlags, JavaTargetAttributes.Builder attributesBuilder); + + /** + * Add extra dependencies for runfiles of a Java binary. + */ + void addDependenciesForRunfiles(RuleContext ruleContext, Builder builder); + + /** + * Determines if we should enforce the use of the :java_launcher target to determine the java + * launcher artifact even if the --java_launcher option was not specified. + */ + boolean forceUseJavaLauncherTarget(RuleContext ruleContext); + + /** + * Add a source artifact to a {@link JavaTargetAttributes.Builder}. It is called when a source + * artifact is processed but is not matched by default patterns in the + * {@link JavaTargetAttributes.Builder#addSourceArtifacts(Iterable)} method. The semantics can + * then detect its custom artifact types and add it to the builder. + */ + void addArtifactToJavaTargetAttribute(JavaTargetAttributes.Builder builder, Artifact srcArtifact); + + /** + * Works on the list of dependencies of a java target to builder the {@link JavaTargetAttributes}. + * This work is performed in {@link JavaCommon} for all java targets. + */ + void commonDependencyProcessing(RuleContext ruleContext, JavaTargetAttributes.Builder attributes, + Collection<? extends TransitiveInfoCollection> deps); + + /** + * Returns an list of {@link ActionInput} that the {@link JavaCompileAction} generates and + * that should be cached. + */ + Collection<ActionInput> getExtraJavaCompileOutputs(PathFragment classDirectory); + + /** + * Takes the path of a Java resource and tries to determine the Java + * root relative path of the resource. + * + * @param path the root relative path of the resource. + * @return the Java root relative path of the resource of the root + * relative path of the resource if no Java root relative path can be + * determined. + */ + PathFragment getJavaResourcePath(PathFragment path); + + /** + * @return a list of extra arguments to appends to the runfiles support. + */ + List<String> getExtraArguments(RuleContext ruleContext, JavaCommon javaCommon); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaSourceJarsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaSourceJarsProvider.java new file mode 100644 index 0000000000..2bb35978ac --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaSourceJarsProvider.java @@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * The collection of source jars from the transitive closure. + */ +@Immutable +public final class JavaSourceJarsProvider implements TransitiveInfoProvider { + + private final NestedSet<Artifact> transitiveSourceJars; + private final ImmutableList<Artifact> sourceJars; + + public JavaSourceJarsProvider(NestedSet<Artifact> transitiveSourceJars, + Iterable<Artifact> sourceJars) { + this.transitiveSourceJars = transitiveSourceJars; + this.sourceJars = ImmutableList.copyOf(sourceJars); + } + + /** + * Returns all the source jars in the transitive closure, that can be reached by a chain of + * JavaSourceJarsProvider instances. + */ + public NestedSet<Artifact> getTransitiveSourceJars() { + return transitiveSourceJars; + } + + /** + * Return the source jars that are to be built when the target is on the command line. + */ + public ImmutableList<Artifact> getSourceJars() { + return sourceJars; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaTargetAttributes.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaTargetAttributes.java new file mode 100644 index 0000000000..a7fc497172 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaTargetAttributes.java @@ -0,0 +1,603 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.IterablesChain; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.cpp.CppFileTypes; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * An object that captures the temporary state we need to pass around while + * the initialization hook for a java rule is running. + */ +public class JavaTargetAttributes { + + private static void checkJar(Artifact classPathEntry) { + if (!JavaSemantics.JAR.matches(classPathEntry.getFilename())) { + throw new IllegalArgumentException( + "not a jar file: " + classPathEntry.prettyPrint()); + } + } + + /** + * A builder class for JavaTargetAttributes. + */ + public static class Builder { + + // The order of source files is important, and there must not be duplicates. + // Unfortunately, there is no interface in Java that represents a collection + // without duplicates that has a stable and deterministic iteration order, + // but is not sorted according to a property of the elements. Thus we are + // stuck with Set. + private final Set<Artifact> sourceFiles = new LinkedHashSet<>(); + private final Set<Artifact> jarFiles = new LinkedHashSet<>(); + private final Set<Artifact> compileTimeJarFiles = new LinkedHashSet<>(); + + private final NestedSetBuilder<Artifact> runtimeClassPath = + NestedSetBuilder.naiveLinkOrder(); + + private final NestedSetBuilder<Artifact> compileTimeClassPath = + NestedSetBuilder.naiveLinkOrder(); + + private final List<Artifact> bootClassPath = new ArrayList<>(); + private final List<Artifact> nativeLibraries = new ArrayList<>(); + + private final Set<Artifact> processorPath = new LinkedHashSet<>(); + private final Set<String> processorNames = new LinkedHashSet<>(); + + private final List<Artifact> resources = new ArrayList<>(); + private final List<Artifact> messages = new ArrayList<>(); + private final List<Artifact> instrumentationMetadata = new ArrayList<>(); + private final List<Artifact> sourceJars = new ArrayList<>(); + + private final List<Artifact> classPathResources = new ArrayList<>(); + + private BuildConfiguration.StrictDepsMode strictJavaDeps = + BuildConfiguration.StrictDepsMode.OFF; + private final List<Artifact> directJars = new ArrayList<>(); + private final List<Artifact> compileTimeDependencyArtifacts = new ArrayList<>(); + private final List<Artifact> runtimeDependencyArtifacts = new ArrayList<>(); + private String ruleKind; + private Label targetLabel; + + private final NestedSetBuilder<Artifact> excludedArtifacts = + NestedSetBuilder.naiveLinkOrder(); + + private boolean built = false; + + private final JavaSemantics semantics; + + public Builder(JavaSemantics semantics) { + this.semantics = semantics; + } + + public Builder addSourceArtifacts(Iterable<Artifact> sourceArtifacts) { + Preconditions.checkArgument(!built); + for (Artifact srcArtifact : sourceArtifacts) { + String srcFilename = srcArtifact.getExecPathString(); + if (JavaSemantics.JAR.matches(srcFilename)) { + runtimeClassPath.add(srcArtifact); + jarFiles.add(srcArtifact); + } else if (JavaSemantics.SOURCE_JAR.matches(srcFilename)) { + sourceJars.add(srcArtifact); + } else if (JavaSemantics.PROPERTIES.matches(srcFilename)) { + // output files of the message compiler + resources.add(srcArtifact); + } else if (JavaSemantics.JAVA_SOURCE.matches(srcFilename)) { + sourceFiles.add(srcArtifact); + } else { + // try specific cases from the semantics. + semantics.addArtifactToJavaTargetAttribute(this, srcArtifact); + } + } + return this; + } + + public Builder addSourceFiles(Iterable<Artifact> sourceFiles) { + Preconditions.checkArgument(!built); + for (Artifact artifact : sourceFiles) { + if (JavaSemantics.JAVA_SOURCE.matches(artifact.getFilename())) { + this.sourceFiles.add(artifact); + } + } + return this; + } + + public Builder merge(JavaCompilationArgs context) { + Preconditions.checkArgument(!built); + addCompileTimeClassPathEntries(context.getCompileTimeJars()); + addRuntimeClassPathEntries(context.getRuntimeJars()); + addInstrumentationMetadataEntries(context.getInstrumentationMetadata()); + return this; + } + + public Builder addSourceJars(Collection<Artifact> sourceJars) { + Preconditions.checkArgument(!built); + this.sourceJars.addAll(sourceJars); + return this; + } + + public Builder addSourceJar(Artifact sourceJar) { + Preconditions.checkArgument(!built); + this.sourceJars.add(sourceJar); + return this; + } + + public Builder addCompileTimeJarFiles(Iterable<Artifact> jars) { + Preconditions.checkArgument(!built); + Iterables.addAll(compileTimeJarFiles, jars); + return this; + } + + public Builder addRuntimeClassPathEntry(Artifact classPathEntry) { + Preconditions.checkArgument(!built); + checkJar(classPathEntry); + runtimeClassPath.add(classPathEntry); + return this; + } + + public Builder addRuntimeClassPathEntries(NestedSet<Artifact> classPathEntries) { + Preconditions.checkArgument(!built); + runtimeClassPath.addTransitive(classPathEntries); + return this; + } + + public Builder addCompileTimeClassPathEntries(NestedSet<Artifact> entries) { + Preconditions.checkArgument(!built); + compileTimeClassPath.addTransitive(entries); + return this; + } + + public Builder addDirectCompileTimeClassPathEntries(Iterable<Artifact> entries) { + Preconditions.checkArgument(!built); + // The other version is preferred as it is more memory-efficient. + for (Artifact classPathEntry : entries) { + compileTimeClassPath.add(classPathEntry); + } + return this; + } + + public Builder setRuleKind(String ruleKind) { + Preconditions.checkArgument(!built); + this.ruleKind = ruleKind; + return this; + } + + public Builder setTargetLabel(Label targetLabel) { + Preconditions.checkArgument(!built); + this.targetLabel = targetLabel; + return this; + } + + /** + * Sets the bootclasspath to be passed to the Java compiler. + * + * <p>If this method is called, then the bootclasspath specified in this JavaTargetAttributes + * instance overrides the default bootclasspath. + */ + public Builder setBootClassPath(List<Artifact> jars) { + Preconditions.checkArgument(!built); + Preconditions.checkArgument(!jars.isEmpty()); + Preconditions.checkState(bootClassPath.isEmpty()); + bootClassPath.addAll(jars); + return this; + } + + public Builder addExcludedArtifacts(NestedSet<Artifact> toExclude) { + Preconditions.checkArgument(!built); + excludedArtifacts.addTransitive(toExclude); + return this; + } + + /** + * Controls how strict the javac compiler will be in checking correct use of + * direct dependencies. + * + * @param strictDeps one of WARN, ERROR or OFF + */ + public Builder setStrictJavaDeps(BuildConfiguration.StrictDepsMode strictDeps) { + Preconditions.checkArgument(!built); + strictJavaDeps = strictDeps; + return this; + } + + /** + * In tandem with strictJavaDeps, directJars represents a subset of the + * compile-time, classpath jars that were provided by direct dependencies. + * When strictJavaDeps is OFF, there is no need to provide directJars, and + * no extra information is passed to javac. When strictJavaDeps is set to + * WARN or ERROR, the compiler command line will include extra flags to + * indicate the warning/error policy and to map the classpath jars to direct + * or transitive dependencies, using the information in directJars. The extra + * flags are formatted like this (same for --indirect_dependency): + * --direct_dependency + * foo/bar/lib.jar + * //java/com/google/foo:bar + * + * @param directJars + */ + public Builder addDirectJars(Iterable<Artifact> directJars) { + Preconditions.checkArgument(!built); + Iterables.addAll(this.directJars, directJars); + return this; + } + + public Builder addCompileTimeDependencyArtifacts(Iterable<Artifact> dependencyArtifacts) { + Preconditions.checkArgument(!built); + Iterables.addAll(this.compileTimeDependencyArtifacts, dependencyArtifacts); + return this; + } + + public Builder addRuntimeDependencyArtifacts(Iterable<Artifact> dependencyArtifacts) { + Preconditions.checkArgument(!built); + Iterables.addAll(this.runtimeDependencyArtifacts, dependencyArtifacts); + return this; + } + + public Builder addInstrumentationMetadataEntries(Iterable<Artifact> metadataEntries) { + Preconditions.checkArgument(!built); + Iterables.addAll(instrumentationMetadata, metadataEntries); + return this; + } + + public Builder addNativeLibrary(Artifact nativeLibrary) { + Preconditions.checkArgument(!built); + String name = nativeLibrary.getFilename(); + if (CppFileTypes.INTERFACE_SHARED_LIBRARY.matches(name)) { + return this; + } + if (!(CppFileTypes.SHARED_LIBRARY.matches(name) + || CppFileTypes.VERSIONED_SHARED_LIBRARY.matches(name))) { + throw new IllegalArgumentException("not a shared library :" + nativeLibrary.prettyPrint()); + } + nativeLibraries.add(nativeLibrary); + return this; + } + + public Builder addNativeLibraries(Iterable<Artifact> nativeLibraries) { + Preconditions.checkArgument(!built); + for (Artifact nativeLibrary : nativeLibraries) { + addNativeLibrary(nativeLibrary); + } + return this; + } + + public Builder addMessages(Collection<Artifact> messages) { + Preconditions.checkArgument(!built); + this.messages.addAll(messages); + return this; + } + + public Builder addMessage(Artifact messagesArtifact) { + Preconditions.checkArgument(!built); + this.messages.add(messagesArtifact); + return this; + } + + public Builder addResources(Collection<Artifact> resources) { + Preconditions.checkArgument(!built); + this.resources.addAll(resources); + return this; + } + + public Builder addResource(Artifact resource) { + Preconditions.checkArgument(!built); + resources.add(resource); + return this; + } + + public Builder addProcessorName(String processor) { + Preconditions.checkArgument(!built); + processorNames.add(processor); + return this; + } + + public Builder addProcessorPath(Iterable<Artifact> jars) { + Preconditions.checkArgument(!built); + Iterables.addAll(processorPath, jars); + return this; + } + + public Builder addClassPathResources(List<Artifact> classPathResources) { + Preconditions.checkArgument(!built); + this.classPathResources.addAll(classPathResources); + return this; + } + + public Builder addClassPathResource(Artifact classPathResource) { + Preconditions.checkArgument(!built); + this.classPathResources.add(classPathResource); + return this; + } + + public JavaTargetAttributes build() { + built = true; + return new JavaTargetAttributes( + sourceFiles, + jarFiles, + compileTimeJarFiles, + runtimeClassPath, + compileTimeClassPath, + bootClassPath, + nativeLibraries, + processorPath, + processorNames, + resources, + messages, + sourceJars, + classPathResources, + directJars, + compileTimeDependencyArtifacts, + ruleKind, + targetLabel, + excludedArtifacts, + strictJavaDeps); + } + + // TODO(bazel-team): Remove these 5 methods. + @Deprecated + public Set<Artifact> getSourceFiles() { + return sourceFiles; + } + + @Deprecated + public boolean hasSourceFiles() { + return !sourceFiles.isEmpty(); + } + + @Deprecated + public List<Artifact> getInstrumentationMetadata() { + return instrumentationMetadata; + } + + @Deprecated + public boolean hasSourceJars() { + return !sourceJars.isEmpty(); + } + + @Deprecated + public boolean hasJarFiles() { + return !jarFiles.isEmpty(); + } + } + + // + // -------------------------- END OF BUILDER CLASS ------------------------- + // + + private final ImmutableSet<Artifact> sourceFiles; + private final ImmutableSet<Artifact> jarFiles; + private final ImmutableSet<Artifact> compileTimeJarFiles; + + private final NestedSet<Artifact> runtimeClassPath; + private final NestedSet<Artifact> compileTimeClassPath; + + private final ImmutableList<Artifact> bootClassPath; + private final ImmutableList<Artifact> nativeLibraries; + + private final ImmutableSet<Artifact> processorPath; + private final ImmutableSet<String> processorNames; + + private final ImmutableList<Artifact> resources; + private final ImmutableList<Artifact> messages; + private final ImmutableList<Artifact> sourceJars; + + private final ImmutableList<Artifact> classPathResources; + + private final ImmutableList<Artifact> directJars; + private final ImmutableList<Artifact> compileTimeDependencyArtifacts; + private final String ruleKind; + private final Label targetLabel; + + private final NestedSet<Artifact> excludedArtifacts; + private final BuildConfiguration.StrictDepsMode strictJavaDeps; + + /** + * Constructor of JavaTargetAttributes. + */ + private JavaTargetAttributes( + Set<Artifact> sourceFiles, + Set<Artifact> jarFiles, + Set<Artifact> compileTimeJarFiles, + NestedSetBuilder<Artifact> runtimeClassPath, + NestedSetBuilder<Artifact> compileTimeClassPath, + List<Artifact> bootClassPath, + List<Artifact> nativeLibraries, + Set<Artifact> processorPath, + Set<String> processorNames, + List<Artifact> resources, + List<Artifact> messages, + List<Artifact> sourceJars, + List<Artifact> classPathResources, + List<Artifact> directJars, + List<Artifact> compileTimeDependencyArtifacts, + String ruleKind, + Label targetLabel, + NestedSetBuilder<Artifact> excludedArtifacts, + BuildConfiguration.StrictDepsMode strictJavaDeps) { + this.sourceFiles = ImmutableSet.copyOf(sourceFiles); + this.jarFiles = ImmutableSet.copyOf(jarFiles); + this.compileTimeJarFiles = ImmutableSet.copyOf(compileTimeJarFiles); + this.runtimeClassPath = runtimeClassPath.build(); + this.compileTimeClassPath = compileTimeClassPath.build(); + this.bootClassPath = ImmutableList.copyOf(bootClassPath); + this.nativeLibraries = ImmutableList.copyOf(nativeLibraries); + this.processorPath = ImmutableSet.copyOf(processorPath); + this.processorNames = ImmutableSet.copyOf(processorNames); + this.resources = ImmutableList.copyOf(resources); + this.messages = ImmutableList.copyOf(messages); + this.sourceJars = ImmutableList.copyOf(sourceJars); + this.classPathResources = ImmutableList.copyOf(classPathResources); + this.directJars = ImmutableList.copyOf(directJars); + this.compileTimeDependencyArtifacts = ImmutableList.copyOf(compileTimeDependencyArtifacts); + this.ruleKind = ruleKind; + this.targetLabel = targetLabel; + this.excludedArtifacts = excludedArtifacts.build(); + this.strictJavaDeps = strictJavaDeps; + } + + public List<Artifact> getDirectJars() { + return directJars; + } + + public List<Artifact> getCompileTimeDependencyArtifacts() { + return compileTimeDependencyArtifacts; + } + + public List<Artifact> getSourceJars() { + return sourceJars; + } + + public Collection<Artifact> getResources() { + return resources; + } + + public List<Artifact> getMessages() { + return messages; + } + + public ImmutableList<Artifact> getClassPathResources() { + return classPathResources; + } + + private NestedSet<Artifact> getExcludedArtifacts() { + return excludedArtifacts; + } + + /** + * Returns the artifacts needed on the runtime classpath of this target. + * + * See also {@link #getRuntimeClassPathForArchive()}. + */ + public NestedSet<Artifact> getRuntimeClassPath() { + return runtimeClassPath; + } + + /** + * Returns the classpath artifacts needed in a deploy jar for this target. + * + * This excludes the artifacts made available by jars in the deployment + * environment. + */ + public Iterable<Artifact> getRuntimeClassPathForArchive() { + Iterable<Artifact> runtimeClasspath = getRuntimeClassPath(); + + if (getExcludedArtifacts().isEmpty()) { + return runtimeClasspath; + } else { + return Iterables.filter(runtimeClasspath, + Predicates.not(Predicates.in(getExcludedArtifacts().toSet()))); + } + } + + public NestedSet<Artifact> getCompileTimeClassPath() { + return compileTimeClassPath; + } + + public ImmutableList<Artifact> getBootClassPath() { + return bootClassPath; + } + + public ImmutableSet<Artifact> getProcessorPath() { + return processorPath; + } + + public Set<Artifact> getSourceFiles() { + return sourceFiles; + } + + public Set<Artifact> getJarFiles() { + return jarFiles; + } + + public Set<Artifact> getCompileTimeJarFiles() { + return compileTimeJarFiles; + } + + public List<Artifact> getNativeLibraries() { + return nativeLibraries; + } + + public Collection<String> getProcessorNames() { + return processorNames; + } + + public boolean hasSourceFiles() { + return !sourceFiles.isEmpty(); + } + + public boolean hasSourceJars() { + return !sourceJars.isEmpty(); + } + + public boolean hasJarFiles() { + return !jarFiles.isEmpty(); + } + + public boolean hasResources() { + return !resources.isEmpty(); + } + + public boolean hasMessages() { + return !messages.isEmpty(); + } + + public boolean hasClassPathResources() { + return !classPathResources.isEmpty(); + } + + public Iterable<Artifact> getArchiveInputs(boolean includeClasspath) { + IterablesChain.Builder<Artifact> inputs = IterablesChain.builder(); + if (includeClasspath) { + inputs.add(ImmutableList.copyOf(getRuntimeClassPathForArchive())); + } + inputs.add(getResources()); + inputs.add(getClassPathResources()); + if (getExcludedArtifacts().isEmpty()) { + return inputs.build(); + } else { + Set<Artifact> excludedJars = Sets.newHashSet(getExcludedArtifacts()); + return ImmutableList.copyOf(Iterables.filter( + inputs.build(), Predicates.not(Predicates.in(excludedJars)))); + } + } + + public String getRuleKind() { + return ruleKind; + } + + public Label getTargetLabel() { + return targetLabel; + } + + public BuildConfiguration.StrictDepsMode getStrictJavaDeps() { + return strictJavaDeps; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java new file mode 100644 index 0000000000..65ed97a8e8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java @@ -0,0 +1,53 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +import java.util.List; + +/** + * Implementation for the {@code java_toolchain} rule. + */ +public final class JavaToolchain implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + final String source = ruleContext.attributes().get("source_version", Type.STRING); + final String target = ruleContext.attributes().get("target_version", Type.STRING); + final String encoding = ruleContext.attributes().get("encoding", Type.STRING); + final List<String> xlint = ruleContext.attributes().get("xlint", Type.STRING_LIST); + final List<String> misc = ruleContext.attributes().get("misc", Type.STRING_LIST); + final JavaConfiguration configuration = ruleContext.getFragment(JavaConfiguration.class); + JavaToolchainProvider provider = new JavaToolchainProvider(source, target, encoding, + ImmutableList.copyOf(xlint), ImmutableList.copyOf(misc), + configuration.getDefaultJavacFlags()); + RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(ruleContext) + .add(JavaToolchainProvider.class, provider) + .setFilesToBuild(new NestedSetBuilder<Artifact>(Order.STABLE_ORDER).build()) + .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY)); + + return builder.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainData.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainData.java new file mode 100644 index 0000000000..0338fb85fe --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainData.java @@ -0,0 +1,55 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Information about the JDK used by the <code>java_*</code> rules. + * + * <p>This class contains the data of the {@code java_toolchain} rules, it is a separate object so + * it can be shared with other tools. + */ +@Immutable +public class JavaToolchainData { + private final ImmutableList<String> options; + + public JavaToolchainData(String source, String target, String encoding, + ImmutableList<String> xlint, ImmutableList<String> misc) { + Builder<String> builder = ImmutableList.<String>builder(); + if (!source.isEmpty()) { + builder.add("-source", source); + } + if (!target.isEmpty()) { + builder.add("-target", target); + } + if (!encoding.isEmpty()) { + builder.add("-encoding", encoding); + } + if (!xlint.isEmpty()) { + builder.add("-Xlint:" + Joiner.on(",").join(xlint)); + } + this.options = builder.addAll(misc).build(); + } + + /** + * @return the list of options as given by the {@code java_toolchain} rule. + */ + public ImmutableList<String> getJavacOptions() { + return options; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java new file mode 100644 index 0000000000..3e210d8fbd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java @@ -0,0 +1,68 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.List; + +/** + * Information about the JDK used by the <code>java_*</code> rules. + */ +@Immutable +public final class JavaToolchainProvider implements TransitiveInfoProvider { + + private final ImmutableList<String> javacOptions; + + public JavaToolchainProvider(String source, String target, String encoding, + ImmutableList<String> xlint, ImmutableList<String> misc, List<String> defaultJavacFlags) { + super(); + // merges the defaultJavacFlags from + // {@link JavaConfiguration} with the flags from the {@code java_toolchain} rule. + JavaToolchainData data = new JavaToolchainData(source, target, encoding, xlint, misc); + this.javacOptions = ImmutableList.<String>builder() + .addAll(data.getJavacOptions()) + .addAll(defaultJavacFlags) + .build(); + } + + /** + * @return the list of default options for the java compiler + */ + public ImmutableList<String> getJavacOptions() { + return javacOptions; + } + + /** + * An helper method to construct the list of javac options. + * + * @param ruleContext The rule context of the current rule. + * @return the list of flags provided by the {@code java_toolchain} rule merged with the one + * provided by the {@link JavaConfiguration} fragment. + */ + public static List<String> getDefaultJavacOptions(RuleContext ruleContext) { + JavaToolchainProvider javaToolchain = + ruleContext.getPrerequisite(":java_toolchain", Mode.TARGET, JavaToolchainProvider.class); + if (javaToolchain == null) { + ruleContext.ruleError("No java_toolchain implicit dependency found. This is probably because" + + " your java configuration is not up-to-date."); + return ImmutableList.of(); + } + return javaToolchain.getJavacOptions(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java new file mode 100644 index 0000000000..16801eecd2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java @@ -0,0 +1,92 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.STRING; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for {@code java_toolchain} + */ +@BlazeRule(name = "java_toolchain", ancestors = {BaseRuleClasses.BaseRule.class}, + factoryClass = JavaToolchain.class) +public final class JavaToolchainRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder.setUndocumented() + /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(source_version) --> + The Java source version (e.g., '6' or '7'). It specifies which set of code structures + are allowed in the Java source code. + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("source_version", STRING).mandatory()) // javac -source flag value. + /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(target_version) --> + The Java target version (e.g., '6' or '7'). It specifies for which Java runtime the class + should be build. + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("target_version", STRING).mandatory()) // javac -target flag value. + /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(encoding) --> + The encoding of the java files (e.g., 'UTF-8'). + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("encoding", STRING).mandatory()) // javac -encoding flag value. + /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(xlint) --> + The list of warning to add or removes from default list. Precedes it with a dash to + removes it. Please see the Javac documentation on the -Xlint options for more information. + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("xlint", STRING_LIST).value(ImmutableList.<String>of())) + /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(xlint) --> + The list of extra arguments for the Java compiler. Please refer to the Java compiler + documentation for the extensive list of possible Java compiler flags. + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("misc", STRING_LIST).value(ImmutableList.<String>of())) + .build(); + } +} +/*<!-- #BLAZE_RULE (NAME = java_toolchain, TYPE = OTHER, FAMILY = Java) --> + +${ATTRIBUTE_SIGNATURE} + +<p> +Specifies the configuration for the Java compiler. Which toolchain to be used can be changed through +the --java_toolchain argument. Normally you should not write those kind of rules unless you want to +tune your Java compiler. +</p> + +${ATTRIBUTE_DEFINITION} + +<h4 id="java_binary_examples">Examples</h4> + +<p>A simple example would be: +</p> + +<pre class="code"> +java_toolchain( + name = "toolchain", + source_version = "7", + target_version = "7", + encoding = "UTF-8", + xlint = [ "classfile", "divzero", "empty", "options", "path" ], + misc = [ "-g" ], +) +</pre> + +<!-- #END_BLAZE_RULE -->*/ diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaUtil.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaUtil.java new file mode 100644 index 0000000000..b2a84ec86f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaUtil.java @@ -0,0 +1,147 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Utility methods for use by Java-related parts of the build system. + */ +public abstract class JavaUtil { + + private JavaUtil() {} + + //---------- Java related methods + + /* + * TODO(bazel-team): (2009) + * + * This way of figuring out Java source roots is basically + * broken. I think we need to support these two use cases: + * (1) A user puts his / her shell into a directory named java. + * (2) Someplace in the tree, there's a package named java. + * + * (1) is more important than (2); and (2) cannot always be guaranteed + * due to sloppy implementations in the past; most notably the old + * tools/boilerplate_rules.mk code for compiling Java. + * + * Basically, to implement correct semantics, we will need to configure + * Java source roots based on the package path, plus some heuristics to + * support legacy code, maybe. + * + * Roughly: + * Given a path, find the source root that applies to it by + * - walk over the elements in the package path + * - add "java", "javatests" to them + * - find the first element that is a maximal prefix to the Java file + * - for experimental, some legacy support that basically has some + * arbitrary padding before the Java sourceroot. + */ + + /** + * Given the filename of a Java source file, returns the name of the toplevel Java class defined + * within it. + */ + public static String getJavaClassName(PathFragment path) { + return FileSystemUtils.removeExtension(path.getBaseName()); + } + + /** + * Find the index of the "java" or "javatests" segment in a Java path fragment + * that precedes the source root. + * + * @param path a Java source dir or file path + * @return the index of the java segment or -1 iff no java segment was found. + */ + private static int javaSegmentIndex(PathFragment path) { + if (path.isAbsolute()) { + throw new IllegalArgumentException("path must not be absolute: '" + path + "'"); + } + return path.getFirstSegment(ImmutableSet.of("java", "javatests")); + } + + /** + * Given the PathFragment of a Java source file, returns the Java package to which it belongs. + */ + public static String getJavaPackageName(PathFragment path) { + int index = javaSegmentIndex(path) + 1; + path = path.subFragment(index, path.segmentCount() - 1); + return path.getPathString().replace('/', '.'); + } + + /** + * Given the PathFragment of a file without extension, returns the + * Java fully qualified class name based on the Java root relative path of the + * specified path or 'null' if no java root can be determined. + * <p> + * For example, "java/foo/bar/wiz" and "javatests/foo/bar/wiz" both + * result in "foo.bar.wiz". + * + * TODO(bazel-team): (2011) We need to have a more robust way to determine the Java root + * of a relative path rather than simply trying to find the "java" or + * "javatests" directory. + */ + public static String getJavaFullClassname(PathFragment path) { + PathFragment javaPath = getJavaPath(path); + if (javaPath != null) { + return javaPath.getPathString().replace('/', '.'); + } + return null; + } + + /** + * Given the PathFragment of a Java source file, returns the Java root relative path or 'null' if + * no java root can be determined. + * + * <p> + * For example, "{workspace}/java/foo/bar/wiz" and "{workspace}/javatests/foo/bar/wiz" + * both result in "foo/bar/wiz". + * + * TODO(bazel-team): (2011) We need to have a more robust way to determine the Java root + * of a relative path rather than simply trying to find the "java" or + * "javatests" directory. + */ + public static PathFragment getJavaPath(PathFragment path) { + int index = javaSegmentIndex(path); + if (index >= 0) { + return path.subFragment(index + 1, path.segmentCount()); + } + return null; + } + + /** + * Given the PathFragment of a Java source file, returns the + * Java root of the specified path or 'null' if no java root can be + * determined. + * <p> + * Example 1: "{workspace}/java/foo/bar/wiz" and "{workspace}/javatests/foo/bar/wiz" + * result in "{workspace}/java" and "{workspace}/javatests" Example 2: + * "java/foo/bar/wiz" and "javatests/foo/bar/wiz" result in "java" and + * "javatests" + * + * TODO(bazel-team): (2011) We need to have a more robust way to determine the Java root + * of a relative path rather than simply trying to find the "java" or + * "javatests" directory. + */ + public static PathFragment getJavaRoot(PathFragment path) { + int index = javaSegmentIndex(path); + if (index >= 0) { + return path.subFragment(0, index + 1); + } + return null; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/Jvm.java b/src/main/java/com/google/devtools/build/lib/rules/java/Jvm.java new file mode 100644 index 0000000000..eda736026a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/Jvm.java @@ -0,0 +1,120 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap.Builder; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * This class represents a Java virtual machine with a host system and a path. + * If the JVM comes from the client, it can optionally also contain a label + * pointing to a target that contains all the necessary files. + */ +@SkylarkModule(name = "jvm", + doc = "A configuration fragment representing the Java virtual machine.") +@Immutable +public final class Jvm extends BuildConfiguration.Fragment { + private final PathFragment javaHome; + private final Label jvmLabel; + + /** + * Creates a Jvm instance. Either the {@code javaHome} parameter is absolute, + * or the {@code jvmLabel} parameter must be non-null. This restriction might + * be lifted in the future. Only the {@code jvmLabel} is optional. + */ + public Jvm(PathFragment javaHome, Label jvmLabel) { + Preconditions.checkArgument(javaHome.isAbsolute() ^ (jvmLabel != null)); + this.javaHome = javaHome; + this.jvmLabel = jvmLabel; + } + + @Override + public String getName() { + return "Jvm"; + } + + @Override + public void addImplicitLabels(Multimap<String, Label> implicitLabels) { + if (jvmLabel != null) { + implicitLabels.put(getName(), jvmLabel); + } + } + + /** + * Returns a path fragment that determines the path to the installation + * directory. It is either absolute or relative to the execution root. + */ + public PathFragment getJavaHome() { + return javaHome; + } + + /** + * Returns the path to the javac binary. + */ + public PathFragment getJavacExecutable() { + return getJavaHome().getRelative("bin/javac"); + } + + /** + * Returns the path to the jar binary. + */ + public PathFragment getJarExecutable() { + return getJavaHome().getRelative("bin/jar"); + } + + /** + * Returns the path to the java binary. + */ + @SkylarkCallable(name = "java_executable", structField = true, + doc = "The the java executable, i.e. bin/java relative to the Java home.") + public PathFragment getJavaExecutable() { + return getJavaHome().getRelative("bin/java"); + } + + /** + * Returns a label. Adding this label to the dependencies of an action that + * depends on this JVM is sufficient to ensure that all the required files are + * present. Can be <code>null</code>, in which case nothing needs to be added + * to the dependencies of an action. We rely on convention to make sure that + * this case works, since we can't know which JVMs are installed on the build host. + */ + public Label getJvmLabel() { + return jvmLabel; + } + + /** + * Returns a string that uniquely identifies the JVM for the life time of this + * Blaze instance. This value is intended for analysis caching, so it need not + * reflect changes in the individual files making up the JVM. + */ + @Override + public String cacheKey() { + return javaHome.getSafePathString(); + } + + @Override + public void addGlobalMakeVariables(Builder<String, String> globalMakeEnvBuilder) { + globalMakeEnvBuilder.put("JAVABASE", getJavaHome().getPathString()); + globalMakeEnvBuilder.put("JAVA", getJavaExecutable().getPathString()); + globalMakeEnvBuilder.put("JAVAC", getJavacExecutable().getPathString()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JvmConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/java/JvmConfigurationLoader.java new file mode 100644 index 0000000000..7483f8827e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JvmConfigurationLoader.java @@ -0,0 +1,163 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.annotations.VisibleForTesting; +import com.google.devtools.build.lib.analysis.RedirectChaser; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.RawAttributeMapper; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.List; + +import javax.annotation.Nullable; + +/** + * A provider to load jvm configurations from the package path. + * + * <p>If the given {@code javaHome} is a label, i.e. starts with {@code "//"}, + * then the loader will look at the target it refers to. If the target is a + * filegroup, then the loader will look in it's srcs for a filegroup that ends + * with {@code -<cpu>}. It will use that filegroup to construct the actual + * {@link Jvm} instance, using the filegroups {@code path} attribute to + * construct the new {@code javaHome} path. + * + * <p>The loader also supports legacy mode, where the JVM can be defined with an abolute path. + */ +public final class JvmConfigurationLoader implements ConfigurationFragmentFactory { + private final boolean forceLegacy; + private final JavaCpuSupplier cpuSupplier; + + public JvmConfigurationLoader(boolean forceLegacy, JavaCpuSupplier cpuSupplier) { + this.forceLegacy = forceLegacy; + this.cpuSupplier = cpuSupplier; + } + + public JvmConfigurationLoader(JavaCpuSupplier cpuSupplier) { + this(/*forceLegacy=*/ false, cpuSupplier); + } + + @Override + public Jvm create(ConfigurationEnvironment env, BuildOptions buildOptions) + throws InvalidConfigurationException { + JavaOptions javaOptions = buildOptions.get(JavaOptions.class); + String javaHome = javaOptions.javaBase; + String cpu = cpuSupplier.getJavaCpu(buildOptions, env); + if (cpu == null) { + return null; + } + + if (!forceLegacy && javaHome.startsWith("//")) { + return createDefault(env, javaHome, cpu); + } else { + return createLegacy(javaHome); + } + } + + @Override + public Class<? extends Fragment> creates() { + return Jvm.class; + } + + @Nullable + private Jvm createDefault(ConfigurationEnvironment lookup, String javaHome, String cpu) + throws InvalidConfigurationException { + try { + Label label = Label.parseAbsolute(javaHome); + label = RedirectChaser.followRedirects(lookup, label, "jdk"); + if (label == null) { + return null; + } + Target javaHomeTarget = lookup.getTarget(label); + if (javaHomeTarget == null) { + return null; + } + if ((javaHomeTarget instanceof Rule) && + "filegroup".equals(((Rule) javaHomeTarget).getRuleClass())) { + RawAttributeMapper javaHomeAttributes = RawAttributeMapper.of((Rule) javaHomeTarget); + if (javaHomeAttributes.isConfigurable("srcs", Type.LABEL_LIST)) { + throw new InvalidConfigurationException("\"srcs\" in " + javaHome + + " is configurable. JAVABASE targets don't support configurable attributes"); + } + List<Label> labels = javaHomeAttributes.get("srcs", Type.LABEL_LIST); + for (Label jvmLabel : labels) { + if (jvmLabel.getName().endsWith("-" + cpu)) { + Target jvmTarget = lookup.getTarget(jvmLabel); + if (jvmTarget == null) { + return null; + } + PathFragment javaHomePath = jvmLabel.getPackageFragment(); + if ((jvmTarget instanceof Rule) && + "filegroup".equals(((Rule) jvmTarget).getRuleClass())) { + RawAttributeMapper jvmTargetAttributes = RawAttributeMapper.of((Rule) jvmTarget); + if (jvmTargetAttributes.isConfigurable("path", Type.STRING)) { + throw new InvalidConfigurationException("\"path\" in " + jvmTarget + + " is configurable. JVM targets don't support configurable attributes"); + } + String path = jvmTargetAttributes.get("path", Type.STRING); + if (path != null) { + javaHomePath = javaHomePath.getRelative(path); + } + } + return new Jvm(javaHomePath, jvmLabel); + } + } + } + throw new InvalidConfigurationException("No JVM target found under " + javaHome + + " that would work for " + cpu); + } catch (NoSuchPackageException e) { + throw new InvalidConfigurationException(e.getMessage(), e); + } catch (NoSuchTargetException e) { + throw new InvalidConfigurationException("No such target: " + e.getMessage(), e); + } catch (SyntaxException e) { + throw new InvalidConfigurationException(e.getMessage(), e); + } + } + + private Jvm createLegacy(String javaHome) + throws InvalidConfigurationException { + if (!javaHome.startsWith("/")) { + throw new InvalidConfigurationException("Illegal javabase value '" + javaHome + + "', javabase must be an absolute path or label"); + } + return new Jvm(new PathFragment(javaHome), null); + } + + /** + * Converts the cpu name to a GNU system name. If the cpu is not a known value, it returns + * <code>"unknown-unknown-linux-gnu"</code>. + */ + @VisibleForTesting + static String convertCpuToGnuSystemName(String cpu) { + if ("piii".equals(cpu)) { + return "i686-unknown-linux-gnu"; + } else if ("k8".equals(cpu)) { + return "x86_64-unknown-linux-gnu"; + } else { + return "unknown-unknown-linux-gnu"; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/MessageBundleProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/MessageBundleProvider.java new file mode 100644 index 0000000000..f78e386497 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/MessageBundleProvider.java @@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Marks configured targets that are able to supply message bundles to their + * dependents. + */ +@Immutable +public final class MessageBundleProvider implements TransitiveInfoProvider { + + private final ImmutableList<Artifact> messages; + + public MessageBundleProvider(ImmutableList<Artifact> messages) { + this.messages = messages; + } + + /** + * The set of XML source files containing the message definitions. + */ + public ImmutableList<Artifact> getMessages() { + return messages; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/NativeLibraryNestedSetBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/java/NativeLibraryNestedSetBuilder.java new file mode 100644 index 0000000000..09ac59fed6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/NativeLibraryNestedSetBuilder.java @@ -0,0 +1,115 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.cpp.CcNativeLibraryProvider; +import com.google.devtools.build.lib.rules.cpp.CppFileTypes; +import com.google.devtools.build.lib.rules.cpp.LinkerInput; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs; +import com.google.devtools.build.lib.util.FileType; + +/** + * A builder that helps construct nested sets of native libraries. + */ +public final class NativeLibraryNestedSetBuilder { + + private final NestedSetBuilder<LinkerInput> builder = NestedSetBuilder.linkOrder(); + + /** + * Build a nested set of native libraries. + */ + public NestedSet<LinkerInput> build() { + return builder.build(); + } + + /** + * Include specified artifacts as native libraries in the nested set. + */ + public NativeLibraryNestedSetBuilder addAll(Iterable<Artifact> deps) { + for (Artifact dep : deps) { + builder.add(new LinkerInputs.SimpleLinkerInput(dep)); + } + return this; + } + + /** + * Include native libraries of specified dependencies into the nested set. + */ + public NativeLibraryNestedSetBuilder addJavaTargets( + Iterable<? extends TransitiveInfoCollection> deps) { + for (TransitiveInfoCollection dep : deps) { + addJavaTarget(dep); + } + return this; + } + + /** + * Include native Java libraries of a specified target into the nested set. + */ + private void addJavaTarget(TransitiveInfoCollection dep) { + JavaNativeLibraryProvider javaProvider = dep.getProvider(JavaNativeLibraryProvider.class); + if (javaProvider != null) { + builder.addTransitive(javaProvider.getTransitiveJavaNativeLibraries()); + return; + } + + CcNativeLibraryProvider ccProvider = dep.getProvider(CcNativeLibraryProvider.class); + if (ccProvider != null) { + builder.addTransitive(ccProvider.getTransitiveCcNativeLibraries()); + return; + } + + addTarget(dep); + } + + /** + * Include native C/C++ libraries of specified dependencies into the nested set. + */ + public NativeLibraryNestedSetBuilder addCcTargets( + Iterable<? extends TransitiveInfoCollection> deps) { + for (TransitiveInfoCollection dep : deps) { + addCcTarget(dep); + } + return this; + } + + /** + * Include native Java libraries of a specified target into the nested set. + */ + private void addCcTarget(TransitiveInfoCollection dep) { + CcNativeLibraryProvider provider = dep.getProvider(CcNativeLibraryProvider.class); + if (provider != null) { + builder.addTransitive(provider.getTransitiveCcNativeLibraries()); + } else { + addTarget(dep); + } + } + + /** + * Include files and genrule artifacts. + */ + private void addTarget(TransitiveInfoCollection dep) { + for (Artifact artifact : FileType.filterList( + dep.getProvider(FileProvider.class).getFilesToBuild(), + CppFileTypes.SHARED_LIBRARY)) { + builder.add(new LinkerInputs.SimpleLinkerInput(artifact)); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/SourcesJavaCompilationArgsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/SourcesJavaCompilationArgsProvider.java new file mode 100644 index 0000000000..ff8507e8a2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/SourcesJavaCompilationArgsProvider.java @@ -0,0 +1,60 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * An interface that marks configured targets that can provide Java compilation arguments through + * the 'srcs' attribute of Java rules. + * + * <p>In a perfect world, this would not be necessary for a million reasons, but + * this world is far from perfect, thus, we need this. + * + * <p>Please do not implement this interface with configured target implementations. + */ +@Immutable +public final class SourcesJavaCompilationArgsProvider implements TransitiveInfoProvider { + private final JavaCompilationArgs javaCompilationArgs; + private final JavaCompilationArgs recursiveJavaCompilationArgs; + + public SourcesJavaCompilationArgsProvider( + JavaCompilationArgs javaCompilationArgs, + JavaCompilationArgs recursiveJavaCompilationArgs) { + this.javaCompilationArgs = javaCompilationArgs; + this.recursiveJavaCompilationArgs = recursiveJavaCompilationArgs; + } + + /** + * Returns non-recursively collected Java compilation information for + * building this target (called when strict_java_deps = 1). + * + * <p>Note that some of the parameters are still collected from the complete + * transitive closure. The non-recursive collection applies mainly to + * compile-time jars. + */ + public JavaCompilationArgs getJavaCompilationArgs() { + return javaCompilationArgs; + } + + /** + * Returns recursively collected Java compilation information for building + * this target (called when strict_java_deps = 0). + */ + public JavaCompilationArgs getRecursiveJavaCompilationArgs() { + return recursiveJavaCompilationArgs; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/WriteBuildInfoPropertiesAction.java b/src/main/java/com/google/devtools/build/lib/rules/java/WriteBuildInfoPropertiesAction.java new file mode 100644 index 0000000000..08dcbba596 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/WriteBuildInfoPropertiesAction.java @@ -0,0 +1,211 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.analysis.BuildInfoHelper; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Key; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +/** + * An action that creates a Java properties file containing the build informations. + */ +public class WriteBuildInfoPropertiesAction extends AbstractFileWriteAction { + private static final String GUID = "922949ca-1391-4046-a300-74810618dcdc"; + + private final ImmutableList<Artifact> valueArtifacts; + private final BuildInfoPropertiesTranslator keyTranslations; + private final boolean includeVolatile; + private final boolean includeNonVolatile; + + private final TimestampFormatter timestampFormatter; + /** + * An interface to format a timestamp. We are using our custom one to avoid external dependency. + */ + public static interface TimestampFormatter { + /** + * Return a human readable string for the given {@code timestamp}. {@code timestamp} is given + * in milliseconds since 1st of January 1970 at 0am UTC. + */ + public String format(long timestamp); + } + + /** + * A wrapper around a {@link Writer} that skips the first line assuming the line is pure ASCII. It + * can be used to strip the timestamp comment that {@link Properties#store(Writer, String)} adds. + */ + @VisibleForTesting + static class StripFirstLineWriter extends Writer { + private final Writer writer; + private boolean newlineFound = false; + + StripFirstLineWriter(OutputStream out) { + this.writer = new OutputStreamWriter(out, UTF_8); + } + + @Override + public void write(char[] cbuf, int off, int len) throws IOException { + if (!newlineFound) { + while (len > 0 && cbuf[off] != '\n') { + off++; + len--; + } + if (len > 0) { + newlineFound = true; + off++; + len--; + } + } + if (len > 0) { + writer.write(cbuf, off, len); + } + } + + @Override + public void flush() throws IOException { + writer.flush(); + } + + @Override + public void close() throws IOException { + writer.close(); + } + + } + + /** + * Creates an action that writes a Java property files with build information. + * + * <p>It reads the set of build info keys from an action context that is usually contributed to + * Blaze by the workspace status module, and the value associated with said keys from the + * workspace status files (stable and volatile) written by the workspace status action. The files + * generated by this action serve as input to the + * {@link com.google.devtools.build.singlejar.SingleJar} program. + * + * <p>Without input artifacts, this action uses redacted build information. + * + * @param inputs Artifacts that contain build information, or an empty collection to use redacted + * build information + * @param output output the properties file Artifact created by this action + * @param keyTranslations how to translates available keys. See + * {@link BuildInfoPropertiesTranslator}. + * @param includeVolatile whether the set of key to write are giving volatile keys or not + * @param includeNonVolatile whether the set of key to write are giving non-volatile keys or not + * @param timestampFormatter formats dates printed in the properties file + */ + public WriteBuildInfoPropertiesAction(Collection<Artifact> inputs, Artifact output, + BuildInfoPropertiesTranslator keyTranslations, boolean includeVolatile, + boolean includeNonVolatile, TimestampFormatter timestampFormatter) { + super(BuildInfoHelper.BUILD_INFO_ACTION_OWNER, inputs, output, /* makeExecutable= */false); + this.keyTranslations = keyTranslations; + this.includeVolatile = includeVolatile; + this.includeNonVolatile = includeNonVolatile; + this.timestampFormatter = timestampFormatter; + valueArtifacts = ImmutableList.copyOf(inputs); + + if (!inputs.isEmpty()) { + // With non-empty inputs we should not generate both volatile and non-volatile data + // in the same properties file. + Preconditions.checkState(includeVolatile ^ includeNonVolatile); + } + Preconditions.checkState( + output.isConstantMetadata() == (includeVolatile && !inputs.isEmpty())); + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, + final Executor executor) { + final long timestamp = System.currentTimeMillis(); + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + WorkspaceStatusAction.Context context = + executor.getContext(WorkspaceStatusAction.Context.class); + Map<String, String> values = new LinkedHashMap<>(); + for (Artifact valueFile : valueArtifacts) { + values.putAll(WorkspaceStatusAction.parseValues(valueFile.getPath())); + } + + Map<String, String> keys = new HashMap<>(); + if (includeVolatile) { + addValues(keys, values, context.getVolatileKeys()); + keys.put("BUILD_TIMESTAMP", Long.toString(timestamp / 1000)); + keys.put("BUILD_TIME", timestampFormatter.format(timestamp)); + } + addValues(keys, values, context.getStableKeys()); + Properties properties = new Properties(); + keyTranslations.translate(keys, properties); + properties.store(new StripFirstLineWriter(out), null); + } + }; + } + + private void addValues(Map<String, String> result, Map<String, String> values, + Map<String, Key> keys) { + boolean redacted = values.isEmpty(); + for (Map.Entry<String, WorkspaceStatusAction.Key> key : keys.entrySet()) { + if (key.getValue().isInLanguage("Java")) { + result.put(key.getKey(), gePropertyValue(values, redacted, key)); + } + } + } + + private static String gePropertyValue(Map<String, String> values, boolean redacted, + Map.Entry<String, WorkspaceStatusAction.Key> key) { + return redacted ? key.getValue().getRedactedValue() + : values.containsKey(key.getKey()) ? values.get(key.getKey()) + : key.getValue().getDefaultValue(); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addString(keyTranslations.computeKey()); + f.addBoolean(includeVolatile); + f.addBoolean(includeNonVolatile); + return f.hexDigestAndReset(); + } + + @Override + public boolean executeUnconditionally() { + return isVolatile(); + } + + @Override + public boolean isVolatile() { + return includeVolatile && !Iterables.isEmpty(getInputs()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ApplicationSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ApplicationSupport.java new file mode 100644 index 0000000000..73554e5d90 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ApplicationSupport.java @@ -0,0 +1,588 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates; +import static com.google.devtools.build.xcode.common.TargetDeviceFamily.UI_DEVICE_FAMILY_VALUES; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.BinaryFileWriteAction; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraActoolArgs; +import com.google.devtools.build.lib.shell.ShellUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.common.InvalidFamilyNameException; +import com.google.devtools.build.xcode.common.Platform; +import com.google.devtools.build.xcode.common.RepeatedFamilyNameException; +import com.google.devtools.build.xcode.common.TargetDeviceFamily; +import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.XcodeprojBuildSetting; + +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Support for application-generating ObjC rules. An application is generally composed of a + * top-level {@link BundleSupport bundle}, potentially signed, as well as some debug information, if + * {@link ObjcConfiguration#generateDebugSymbols() requested}. + * + * <p>Contains actions, validation logic and provider value generation. + * + * <p>Methods on this class can be called in any order without impacting the result. + */ +public final class ApplicationSupport { + + /** + * Template for the containing application folder. + */ + public static final SafeImplicitOutputsFunction IPA = fromTemplates("%{name}.ipa"); + + @VisibleForTesting + static final String NO_ASSET_CATALOG_ERROR_FORMAT = + "a value was specified (%s), but this app does not have any asset catalogs"; + @VisibleForTesting + static final String INVALID_FAMILIES_ERROR = + "Expected one or two strings from the list 'iphone', 'ipad'"; + @VisibleForTesting + static final String DEVICE_NO_PROVISIONING_PROFILE = + "Provisioning profile must be set for device build"; + + @VisibleForTesting + static final String PROVISIONING_PROFILE_BUNDLE_FILE = "embedded.mobileprovision"; + + private final Attributes attributes; + private final BundleSupport bundleSupport; + private final RuleContext ruleContext; + private final Bundling bundling; + private final ObjcProvider objcProvider; + private final LinkedBinary linkedBinary; + private final ImmutableSet<TargetDeviceFamily> families; + private final IntermediateArtifacts intermediateArtifacts; + + /** + * Indicator as to whether this rule generates a binary directly or whether only dependencies + * should be considered. + */ + enum LinkedBinary { + /** + * This rule generates its own binary which should be included as well as dependency-generated + * binaries. + */ + LOCAL_AND_DEPENDENCIES, + + /** + * This rule does not generate its own binary, only consider binaries from dependencies. + */ + DEPENDENCIES_ONLY + } + + /** + * Creates a new application support within the given rule context. + * + * @param ruleContext context for the application-generating rule + * @param objcProvider provider containing all dependencies' information as well as some of this + * rule's + * @param optionsProvider provider containing options and plist settings for this rule and its + * dependencies + * @param linkedBinary whether to look for a linked binary from this rule and dependencies or just + * the latter + */ + ApplicationSupport( + RuleContext ruleContext, ObjcProvider objcProvider, OptionsProvider optionsProvider, + LinkedBinary linkedBinary) { + this.linkedBinary = linkedBinary; + this.attributes = new Attributes(ruleContext); + this.ruleContext = ruleContext; + this.objcProvider = objcProvider; + this.families = ImmutableSet.copyOf(attributes.families()); + this.intermediateArtifacts = ObjcRuleClasses.intermediateArtifacts(ruleContext); + bundling = bundling(ruleContext, objcProvider, optionsProvider); + bundleSupport = new BundleSupport(ruleContext, families, bundling, extraActoolArgs()); + } + + /** + * Validates application-related attributes set on this rule and registers any errors with the + * rule context. + * + * @return this application support + */ + ApplicationSupport validateAttributes() { + bundleSupport.validateAttributes(); + + // No asset catalogs. That means you cannot specify app_icon or + // launch_image attributes, since they must not exist. However, we don't + // run actool in this case, which means it does not do validity checks, + // and we MUST raise our own error somehow... + if (!objcProvider.hasAssetCatalogs()) { + if (attributes.appIcon() != null) { + ruleContext.attributeError("app_icon", + String.format(NO_ASSET_CATALOG_ERROR_FORMAT, attributes.appIcon())); + } + if (attributes.launchImage() != null) { + ruleContext.attributeError("launch_image", + String.format(NO_ASSET_CATALOG_ERROR_FORMAT, attributes.launchImage())); + } + } + + if (families.isEmpty()) { + ruleContext.attributeError("families", INVALID_FAMILIES_ERROR); + } + + return this; + } + + /** + * Registers actions required to build an application. This includes any + * {@link BundleSupport#registerActions(ObjcProvider) bundle} and bundle merge actions, signing + * this application if appropriate and combining several single-architecture binaries into one + * multi-architecture binary. + * + * @return this application support + */ + ApplicationSupport registerActions() { + bundleSupport.registerActions(objcProvider); + + registerCombineArchitecturesAction(); + + ObjcConfiguration objcConfiguration = ObjcRuleClasses.objcConfiguration(ruleContext); + Artifact ipaOutput = ruleContext.getImplicitOutputArtifact(IPA); + + Artifact maybeSignedIpa; + if (objcConfiguration.getPlatform() == Platform.SIMULATOR) { + maybeSignedIpa = ipaOutput; + } else if (attributes.provisioningProfile() == null) { + throw new IllegalStateException(DEVICE_NO_PROVISIONING_PROFILE); + } else { + maybeSignedIpa = registerBundleSigningActions(ipaOutput); + } + + BundleMergeControlBytes bundleMergeControlBytes = new BundleMergeControlBytes( + bundling, maybeSignedIpa, objcConfiguration, families); + registerBundleMergeActions( + maybeSignedIpa, bundling.getBundleContentArtifacts(), bundleMergeControlBytes); + + return this; + } + + private Artifact registerBundleSigningActions(Artifact ipaOutput) { + PathFragment entitlementsDirectory = ruleContext.getUniqueDirectory("entitlements"); + Artifact teamPrefixFile = ruleContext.getRelatedArtifact( + entitlementsDirectory, ".team_prefix_file"); + registerExtractTeamPrefixAction(teamPrefixFile); + + Artifact entitlementsNeedingSubstitution = attributes.entitlements(); + if (entitlementsNeedingSubstitution == null) { + entitlementsNeedingSubstitution = ruleContext.getRelatedArtifact( + entitlementsDirectory, ".entitlements_with_variables"); + registerExtractEntitlementsAction(entitlementsNeedingSubstitution); + } + Artifact entitlements = ruleContext.getRelatedArtifact( + entitlementsDirectory, ".entitlements"); + registerEntitlementsVariableSubstitutionAction( + entitlementsNeedingSubstitution, entitlements, teamPrefixFile); + Artifact ipaUnsigned = ObjcRuleClasses.artifactByAppendingToRootRelativePath( + ruleContext, ipaOutput.getExecPath(), ".unsigned"); + registerSignBundleAction(entitlements, ipaOutput, ipaUnsigned); + return ipaUnsigned; + } + + /** + * Adds bundle- and application-related settings to the given Xcode provider builder. + * + * @return this application support + */ + ApplicationSupport addXcodeSettings(XcodeProvider.Builder xcodeProviderBuilder) { + bundleSupport.addXcodeSettings(xcodeProviderBuilder); + xcodeProviderBuilder.addXcodeprojBuildSettings(buildSettings()); + + return this; + } + + /** + * Adds any files to the given nested set builder that should be built if this application is the + * top level target in a blaze invocation. + * + * @return this application support + */ + ApplicationSupport addFilesToBuild(NestedSetBuilder<Artifact> filesToBuild) { + NestedSetBuilder<Artifact> debugSymbolBuilder = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(objcProvider.get(ObjcProvider.DEBUG_SYMBOLS)); + + if (linkedBinary == LinkedBinary.LOCAL_AND_DEPENDENCIES + && ObjcRuleClasses.objcConfiguration(ruleContext).generateDebugSymbols()) { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + debugSymbolBuilder.add(intermediateArtifacts.dsymPlist()) + .add(intermediateArtifacts.dsymSymbol()) + .add(intermediateArtifacts.breakpadSym()); + } + + filesToBuild.add(ruleContext.getImplicitOutputArtifact(ApplicationSupport.IPA)) + // TODO(bazel-team): Fat binaries may require some merging of these file rather than just + // making them available. + .addTransitive(debugSymbolBuilder.build()); + return this; + } + + /** + * Creates the {@link XcTestAppProvider} that can be used if this application is used as an + * {@code xctest_app}. + */ + XcTestAppProvider xcTestAppProvider() { + // We want access to #import-able things from our test rig's dependency graph, but we don't + // want to link anything since that stuff is shared automatically by way of the + // -bundle_loader linker flag. + ObjcProvider partialObjcProvider = new ObjcProvider.Builder() + .addTransitiveAndPropagate(ObjcProvider.HEADER, objcProvider) + .addTransitiveAndPropagate(ObjcProvider.INCLUDE, objcProvider) + .addTransitiveAndPropagate(ObjcProvider.SDK_DYLIB, objcProvider) + .addTransitiveAndPropagate(ObjcProvider.SDK_FRAMEWORK, objcProvider) + .addTransitiveAndPropagate(ObjcProvider.WEAK_SDK_FRAMEWORK, objcProvider) + .addTransitiveAndPropagate(ObjcProvider.FRAMEWORK_DIR, objcProvider) + .addTransitiveAndPropagate(ObjcProvider.FRAMEWORK_FILE, objcProvider) + .build(); + // TODO(bazel-team): Handle the FRAMEWORK_DIR key properly. We probably want to add it to + // framework search paths, but not actually link it with the -framework flag. + return new XcTestAppProvider(intermediateArtifacts.singleArchitectureBinary(), + ruleContext.getImplicitOutputArtifact(IPA), partialObjcProvider); + } + + private ExtraActoolArgs extraActoolArgs() { + ImmutableList.Builder<String> extraArgs = ImmutableList.builder(); + if (attributes.appIcon() != null) { + extraArgs.add("--app-icon", attributes.appIcon()); + } + if (attributes.launchImage() != null) { + extraArgs.add("--launch-image", attributes.launchImage()); + } + return new ExtraActoolArgs(extraArgs.build()); + } + + private Bundling bundling( + RuleContext ruleContext, ObjcProvider objcProvider, OptionsProvider optionsProvider) { + ImmutableList<BundleableFile> extraBundleFiles; + ObjcConfiguration objcConfiguration = ObjcRuleClasses.objcConfiguration(ruleContext); + if (objcConfiguration.getPlatform() == Platform.DEVICE) { + extraBundleFiles = ImmutableList.of(new BundleableFile( + attributes.provisioningProfile(), + PROVISIONING_PROFILE_BUNDLE_FILE)); + } else { + extraBundleFiles = ImmutableList.of(); + } + + return new Bundling.Builder() + .setName(ruleContext.getLabel().getName()) + .setBundleDirSuffix(".app") + .setExtraBundleFiles(extraBundleFiles) + .setObjcProvider(objcProvider) + .setInfoplistMerging( + BundleSupport.infoPlistMerging(ruleContext, objcProvider, optionsProvider)) + .setIntermediateArtifacts(intermediateArtifacts) + .build(); + } + + private void registerCombineArchitecturesAction() { + Artifact resultingLinkedBinary = intermediateArtifacts.combinedArchitectureBinary(".app"); + NestedSet<Artifact> linkedBinaries = linkedBinaries(); + + ruleContext.registerAction(ObjcActionsBuilder.spawnOnDarwinActionBuilder() + .setMnemonic("ObjcCombiningArchitectures") + .addTransitiveInputs(linkedBinaries) + .addOutput(resultingLinkedBinary) + .setExecutable(ObjcActionsBuilder.LIPO) + .setCommandLine(CustomCommandLine.builder() + .addExecPaths("-create", linkedBinaries) + .addExecPath("-o", resultingLinkedBinary) + .build()) + .build(ruleContext)); + } + + private NestedSet<Artifact> linkedBinaries() { + NestedSetBuilder<Artifact> linkedBinariesBuilder = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(attributes.dependentLinkedBinaries()); + if (linkedBinary == LinkedBinary.LOCAL_AND_DEPENDENCIES) { + linkedBinariesBuilder.add(intermediateArtifacts.singleArchitectureBinary()); + } + return linkedBinariesBuilder.build(); + } + + /** Returns this target's Xcode build settings. */ + private Iterable<XcodeprojBuildSetting> buildSettings() { + ImmutableList.Builder<XcodeprojBuildSetting> buildSettings = new ImmutableList.Builder<>(); + if (attributes.appIcon() != null) { + buildSettings.add(XcodeprojBuildSetting.newBuilder() + .setName("ASSETCATALOG_COMPILER_APPICON_NAME") + .setValue(attributes.appIcon()) + .build()); + } + if (attributes.launchImage() != null) { + buildSettings.add(XcodeprojBuildSetting.newBuilder() + .setName("ASSETCATALOG_COMPILER_LAUNCHIMAGE_NAME") + .setValue(attributes.launchImage()) + .build()); + } + + // Convert names to a sequence containing "1" and/or "2" for iPhone and iPad, respectively. + Iterable<Integer> familyIndexes = + families.isEmpty() ? ImmutableList.<Integer>of() : UI_DEVICE_FAMILY_VALUES.get(families); + buildSettings.add(XcodeprojBuildSetting.newBuilder() + .setName("TARGETED_DEVICE_FAMILY") + .setValue(Joiner.on(',').join(familyIndexes)) + .build()); + + Artifact entitlements = attributes.entitlements(); + if (entitlements != null) { + buildSettings.add(XcodeprojBuildSetting.newBuilder() + .setName("CODE_SIGN_ENTITLEMENTS") + .setValue("$(WORKSPACE_ROOT)/" + entitlements.getExecPathString()) + .build()); + } + + return buildSettings.build(); + } + + private ApplicationSupport registerSignBundleAction( + Artifact entitlements, Artifact ipaOutput, Artifact ipaUnsigned) { + // TODO(bazel-team): Support variable substitution + ruleContext.registerAction(ObjcActionsBuilder.spawnOnDarwinActionBuilder() + .setMnemonic("IosSignBundle") + .setProgressMessage("Signing iOS bundle: " + ruleContext.getLabel()) + .setExecutable(new PathFragment("/bin/bash")) + .addArgument("-c") + // TODO(bazel-team): Support --resource-rules for resources + .addArgument("set -e && " + + "t=$(mktemp -d -t signing_intermediate) && " + // Get an absolute path since we need to cd into the temp directory for zip. + + "signed_ipa=${PWD}/" + ipaOutput.getExecPathString() + " && " + + "unzip -qq " + ipaUnsigned.getExecPathString() + " -d ${t} && " + + codesignCommand( + attributes.provisioningProfile(), + entitlements, + String.format("${t}/Payload/%s.app", ruleContext.getLabel().getName())) + " && " + // Using zip since we need to preserve permissions + + "cd \"${t}\" && /usr/bin/zip -q -r \"${signed_ipa}\" .") + .addInput(ipaUnsigned) + .addInput(attributes.provisioningProfile()) + .addInput(entitlements) + .addOutput(ipaOutput) + .build(ruleContext)); + + return this; + } + + private void registerBundleMergeActions(Artifact ipaUnsigned, + NestedSet<Artifact> bundleContentArtifacts, BundleMergeControlBytes controlBytes) { + Artifact bundleMergeControlArtifact = + ObjcRuleClasses.artifactByAppendingToBaseName(ruleContext, ".ipa-control"); + + ruleContext.registerAction( + new BinaryFileWriteAction( + ruleContext.getActionOwner(), bundleMergeControlArtifact, controlBytes, + /*makeExecutable=*/false)); + + ruleContext.registerAction(new SpawnAction.Builder() + .setMnemonic("IosBundle") + .setProgressMessage("Bundling iOS application: " + ruleContext.getLabel()) + .setExecutable(attributes.bundleMergeExecutable()) + .addInputArgument(bundleMergeControlArtifact) + .addTransitiveInputs(bundleContentArtifacts) + .addOutput(ipaUnsigned) + .build(ruleContext)); + } + + private void registerExtractTeamPrefixAction(Artifact teamPrefixFile) { + ruleContext.registerAction(ObjcActionsBuilder.spawnOnDarwinActionBuilder() + .setMnemonic("ExtractIosTeamPrefix") + .setExecutable(new PathFragment("/bin/bash")) + .addArgument("-c") + .addArgument("set -e &&" + + " PLIST=$(" + extractPlistCommand(attributes.provisioningProfile()) + ") && " + + // We think PlistBuddy uses PRead internally to seek through the file. Or possibly + // mmaps the file. Or something similar. + // + // Pipe FDs do not support PRead or mmap, though. + // + // <<< however does something magical like write to a temporary file or something + // like that internally, which means that this Just Works. + + " PREFIX=$(/usr/libexec/PlistBuddy -c 'Print ApplicationIdentifierPrefix:0'" + + " /dev/stdin <<< \"${PLIST}\") && " + + " echo ${PREFIX} > " + teamPrefixFile.getExecPathString()) + .addInput(attributes.provisioningProfile()) + .addOutput(teamPrefixFile) + .build(ruleContext)); + } + + private ApplicationSupport registerExtractEntitlementsAction(Artifact entitlements) { + // See Apple Glossary (http://goo.gl/EkhXOb) + // An Application Identifier is constructed as: TeamID.BundleID + // TeamID is extracted from the provisioning profile. + // BundleID consists of a reverse-DNS string to identify the app, where the last component + // is the application name, and is specified as an attribute. + + ruleContext.registerAction(ObjcActionsBuilder.spawnOnDarwinActionBuilder() + .setMnemonic("ExtractIosEntitlements") + .setProgressMessage("Extracting entitlements: " + ruleContext.getLabel()) + .setExecutable(new PathFragment("/bin/bash")) + .addArgument("-c") + .addArgument("set -e && " + + "PLIST=$(" + + extractPlistCommand(attributes.provisioningProfile()) + ") && " + + // We think PlistBuddy uses PRead internally to seek through the file. Or possibly + // mmaps the file. Or something similar. + // + // Pipe FDs do not support PRead or mmap, though. + // + // <<< however does something magical like write to a temporary file or something + // like that internally, which means that this Just Works. + + + "/usr/libexec/PlistBuddy -x -c 'Print Entitlements' /dev/stdin <<< \"${PLIST}\" " + + "> " + entitlements.getExecPathString()) + .addInput(attributes.provisioningProfile()) + .addOutput(entitlements) + .build(ruleContext)); + + return this; + } + + private void registerEntitlementsVariableSubstitutionAction(Artifact in, Artifact out, + Artifact prefix) { + String escapedBundleId = ShellUtils.shellEscape(attributes.bundleId()); + ruleContext.registerAction(new SpawnAction.Builder() + .setMnemonic("SubstituteIosEntitlements") + .setExecutable(new PathFragment("/bin/bash")) + .addArgument("-c") + .addArgument("set -e && " + + "PREFIX=\"$(cat " + prefix.getExecPathString() + ")\" && " + + "sed " + in.getExecPathString() + " " + // Replace .* from default entitlements file with bundle ID where suitable. + + "-e \"s#${PREFIX}\\.\\*#${PREFIX}." + escapedBundleId + "#g\" " + + // Replace some variables that people put in their own entitlements files + + "-e \"s#\\$(AppIdentifierPrefix)#${PREFIX}.#g\" " + + "-e \"s#\\$(CFBundleIdentifier)#" + escapedBundleId + "#g\" " + + + "> " + out.getExecPathString()) + .addInput(in) + .addInput(prefix) + .addOutput(out) + .build(ruleContext)); + } + + + private String extractPlistCommand(Artifact provisioningProfile) { + return "security cms -D -i " + ShellUtils.shellEscape(provisioningProfile.getExecPathString()); + } + + private String codesignCommand( + Artifact provisioningProfile, Artifact entitlements, String appDir) { + String fingerprintCommand = + "/usr/libexec/PlistBuddy -c 'Print DeveloperCertificates:0' /dev/stdin <<< " + + "$(" + extractPlistCommand(provisioningProfile) + ") | " + + "openssl x509 -inform DER -noout -fingerprint | " + + "cut -d= -f2 | sed -e 's#:##g'"; + return String.format( + "/usr/bin/codesign --force --sign $(%s) --entitlements %s %s", + fingerprintCommand, + entitlements.getExecPathString(), + appDir); + } + + /** + * Logic to access attributes required by application support. Attributes are required and + * guaranteed to return a value or throw unless they are annotated with {@link Nullable} in which + * case they can return {@code null} if no value is defined. + */ + private static class Attributes { + private final RuleContext ruleContext; + + private Attributes(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + @Nullable + String appIcon() { + return stringAttribute("app_icon"); + } + + @Nullable + String launchImage() { + return stringAttribute("launch_image"); + } + + @Nullable + Artifact provisioningProfile() { + return ruleContext.getPrerequisiteArtifact("provisioning_profile", Mode.TARGET); + } + + /** + * Returns the value of the {@code families} attribute in a form that is more useful than a list + * of strings. Returns an empty set for any invalid {@code families} attribute value, including + * an empty list. + */ + Set<TargetDeviceFamily> families() { + List<String> rawFamilies = ruleContext.attributes().get("families", Type.STRING_LIST); + try { + return TargetDeviceFamily.fromNamesInRule(rawFamilies); + } catch (InvalidFamilyNameException | RepeatedFamilyNameException e) { + return ImmutableSet.of(); + } + } + + @Nullable + Artifact entitlements() { + return ruleContext.getPrerequisiteArtifact("entitlements", Mode.TARGET); + } + + NestedSet<? extends Artifact> dependentLinkedBinaries() { + if (ruleContext.attributes().getAttributeDefinition("binary") == null) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + return ruleContext.getPrerequisite("binary", Mode.TARGET, ObjcProvider.class) + .get(ObjcProvider.LINKED_BINARY); + } + + FilesToRunProvider bundleMergeExecutable() { + return checkNotNull(ruleContext.getExecutablePrerequisite("$bundlemerge", Mode.HOST)); + } + + String bundleId() { + return checkNotNull(stringAttribute("bundle_id")); + } + + @Nullable + private String stringAttribute(String attribute) { + String value = ruleContext.attributes().get(attribute, Type.STRING); + return value.isEmpty() ? null : value; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ArtifactListAttribute.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ArtifactListAttribute.java new file mode 100644 index 0000000000..d6c993b80f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ArtifactListAttribute.java @@ -0,0 +1,45 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; + +import java.util.Locale; + +/** + * Attributes containing one or more labels. + */ +public enum ArtifactListAttribute { + BUNDLE_IMPORTS; + + public String attrName() { + return name().toLowerCase(Locale.US); + } + + /** + * The artifacts specified by this attribute on the given rule. Returns an empty sequence if the + * attribute is omitted or not available on the rule type. + */ + public Iterable<Artifact> get(RuleContext context) { + if (context.attributes().getAttributeDefinition(attrName()) == null) { + return ImmutableList.of(); + } else { + return context.getPrerequisiteArtifacts(attrName(), Mode.TARGET).list(); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/BundleMergeControlBytes.java b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleMergeControlBytes.java new file mode 100644 index 0000000000..695dffc4d6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleMergeControlBytes.java @@ -0,0 +1,121 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_FILE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.NESTED_BUNDLE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCDATAMODEL; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.io.ByteSource; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos; +import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.Control; +import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.MergeZip; +import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.VariableSubstitution; +import com.google.devtools.build.xcode.common.TargetDeviceFamily; + +import java.io.InputStream; +import java.util.Map; + +/** + * A byte source that can be used to generate a control file for the tool: + * {@code //java/com/google/devtools/build/xcode/bundlemerge}. Note that this generates the control + * proto and bytes on-the-fly rather than eagerly. This is to prevent a copy of the bundle files and + * .xcdatamodels from being stored for each {@code objc_binary} (or any bundle) being built. + */ +final class BundleMergeControlBytes extends ByteSource { + private final Bundling rootBundling; + private final Artifact mergedIpa; + private final ObjcConfiguration objcConfiguration; + private final ImmutableSet<TargetDeviceFamily> families; + + public BundleMergeControlBytes( + Bundling rootBundling, Artifact mergedIpa, ObjcConfiguration objcConfiguration, + ImmutableSet<TargetDeviceFamily> families) { + this.rootBundling = Preconditions.checkNotNull(rootBundling); + this.mergedIpa = Preconditions.checkNotNull(mergedIpa); + this.objcConfiguration = Preconditions.checkNotNull(objcConfiguration); + this.families = Preconditions.checkNotNull(families); + } + + @Override + public InputStream openStream() { + return control("Payload/", "Payload/", rootBundling) + .toByteString() + .newInput(); + } + + private Control control(String mergeZipPrefix, String bundleDirPrefix, Bundling bundling) { + ObjcProvider objcProvider = bundling.getObjcProvider(); + String bundleDir = bundleDirPrefix + bundling.getBundleDir(); + mergeZipPrefix += bundling.getBundleDir() + "/"; + + BundleMergeProtos.Control.Builder control = BundleMergeProtos.Control.newBuilder() + .addAllBundleFile(BundleableFile.toBundleFiles(bundling.getExtraBundleFiles())) + .addAllBundleFile(BundleableFile.toBundleFiles(objcProvider.get(BUNDLE_FILE))) + .addAllSourcePlistFile(Artifact.toExecPaths( + bundling.getInfoplistMerging().getPlistWithEverything().asSet())) + // TODO(bazel-team): Add rule attribute for specifying targeted device family + .setMinimumOsVersion(objcConfiguration.getMinimumOs()) + .setSdkVersion(objcConfiguration.getIosSdkVersion()) + .setPlatform(objcConfiguration.getPlatform().name()) + .setBundleRoot(bundleDir); + + for (Artifact mergeZip : bundling.getMergeZips()) { + control.addMergeZip(MergeZip.newBuilder() + .setEntryNamePrefix(mergeZipPrefix) + .setSourcePath(mergeZip.getExecPathString()) + .build()); + } + + for (Xcdatamodel datamodel : objcProvider.get(XCDATAMODEL)) { + control.addMergeZip(MergeZip.newBuilder() + .setEntryNamePrefix(mergeZipPrefix) + .setSourcePath(datamodel.getOutputZip().getExecPathString()) + .build()); + } + for (TargetDeviceFamily targetDeviceFamily : families) { + control.addTargetDeviceFamily(targetDeviceFamily.name()); + } + + Map<String, String> variableSubstitutions = bundling.variableSubstitutions(); + for (String variable : variableSubstitutions.keySet()) { + control.addVariableSubstitution(VariableSubstitution.newBuilder() + .setName(variable) + .setValue(variableSubstitutions.get(variable)) + .build()); + } + + control.setOutFile(mergedIpa.getExecPathString()); + + for (Artifact linkedBinary : bundling.getCombinedArchitectureBinary().asSet()) { + control + .addBundleFile(BundleMergeProtos.BundleFile.newBuilder() + .setSourceFile(linkedBinary.getExecPathString()) + .setBundlePath(bundling.getName()) + .setExternalFileAttribute(BundleableFile.EXECUTABLE_EXTERNAL_FILE_ATTRIBUTE) + .build()) + .setExecutableName(bundling.getName()); + } + + for (Bundling nestedBundling : bundling.getObjcProvider().get(NESTED_BUNDLE)) { + control.addNestedBundle(control(mergeZipPrefix, "", nestedBundling)); + } + + return control.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/BundleSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleSupport.java new file mode 100644 index 0000000000..5434ff7707 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleSupport.java @@ -0,0 +1,184 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraActoolArgs; +import com.google.devtools.build.lib.rules.objc.XcodeProvider.Builder; +import com.google.devtools.build.xcode.common.TargetDeviceFamily; + +import java.util.Set; + +/** + * Support for generating iOS bundles which contain metadata (a plist file), assets, resources and + * optionally a binary: registers actions that assemble resources and merge plists, provides data + * to providers and validates bundle-related attributes. + * + * <p>Methods on this class can be called in any order without impacting the result. + */ +final class BundleSupport { + + @VisibleForTesting + static final String NO_INFOPLIST_ERROR = "An infoplist must be specified either in the " + + "'infoplist' attribute or via the 'options' attribute, but none was found"; + + private final RuleContext ruleContext; + private final Set<TargetDeviceFamily> targetDeviceFamilies; + private final ExtraActoolArgs extraActoolArgs; + private final Bundling bundling; + + /** + * Returns merging instructions for a bundle's {@code Info.plist}. + * + * @param ruleContext context this bundle is constructed in + * @param objcProvider provider containing all dependencies' information as well as some of this + * rule's + * @param optionsProvider provider containing options and plist settings for this rule and its + * dependencies + */ + static InfoplistMerging infoPlistMerging(RuleContext ruleContext, + ObjcProvider objcProvider, OptionsProvider optionsProvider) { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + + return new InfoplistMerging.Builder(ruleContext) + .setIntermediateArtifacts(intermediateArtifacts) + .setInputPlists(NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(optionsProvider.getInfoplists()) + .addAll(actoolPartialInfoplist(ruleContext, objcProvider).asSet()) + .build()) + .setPlmerge(ruleContext.getExecutablePrerequisite("$plmerge", Mode.HOST)) + .build(); + } + + /** + * Creates a new bundle support with no special {@code actool} arguments. + * + * @param ruleContext context this bundle is constructed in + * @param targetDeviceFamilies device families used in asset catalogue construction + * @param bundling bundle information as configured for this rule + */ + public BundleSupport( + RuleContext ruleContext, Set<TargetDeviceFamily> targetDeviceFamilies, Bundling bundling) { + this(ruleContext, targetDeviceFamilies, bundling, new ExtraActoolArgs()); + } + + /** + * Creates a new bundle support. + * + * @param ruleContext context this bundle is constructed in + * @param targetDeviceFamilies device families used in asset catalogue construction + * @param bundling bundle information as configured for this rule + * @param extraActoolArgs any additional parameters to be used for invoking {@code actool} + */ + public BundleSupport(RuleContext ruleContext, Set<TargetDeviceFamily> targetDeviceFamilies, + Bundling bundling, ExtraActoolArgs extraActoolArgs) { + this.ruleContext = ruleContext; + this.targetDeviceFamilies = targetDeviceFamilies; + this.extraActoolArgs = extraActoolArgs; + this.bundling = bundling; + } + + /** + * Registers actions required for constructing this bundle, namely merging all involved {@code + * Info.plist} files and generating asset catalogues. + * + * @param objcProvider source of information from this rule's attributes and its dependencies + * + * @return this bundle support + */ + BundleSupport registerActions(ObjcProvider objcProvider) { + registerMergeInfoplistAction(); + registerActoolActionIfNecessary(objcProvider); + + return this; + } + + /** + * Adds any Xcode settings related to this bundle to the given provider builder. + * + * @return this bundle support + */ + BundleSupport addXcodeSettings(Builder xcodeProviderBuilder) { + xcodeProviderBuilder.setInfoplistMerging(bundling.getInfoplistMerging()); + return this; + } + + /** + * Validates any rule attributes and dependencies related to this bundle. + * + * @return this bundle support + */ + BundleSupport validateAttributes() { + if (bundling.getInfoplistMerging().getInputPlists().isEmpty()) { + ruleContext.ruleError(NO_INFOPLIST_ERROR); + } + return this; + } + + private void registerMergeInfoplistAction() { + // TODO(bazel-team): Move action implementation from InfoplistMerging to this class. + ruleContext.registerAction(bundling.getInfoplistMerging().getMergeAction()); + } + + private void registerActoolActionIfNecessary(ObjcProvider objcProvider) { + Optional<Artifact> actoolzipOutput = bundling.getActoolzipOutput(); + if (!actoolzipOutput.isPresent()) { + return; + } + + ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext); + + Artifact actoolPartialInfoplist = actoolPartialInfoplist(ruleContext, objcProvider).get(); + actionsBuilder.registerActoolzipAction( + new ObjcRuleClasses.Tools(ruleContext), + objcProvider, + actoolzipOutput.get(), + new ObjcActionsBuilder.ExtraActoolOutputs(actoolPartialInfoplist), + new ExtraActoolArgs( + new ImmutableList.Builder<String>() + .addAll(extraActoolArgs) + .add("--output-partial-info-plist", actoolPartialInfoplist.getExecPathString()) + .build()), + targetDeviceFamilies); + } + + /** + * Returns the artifact that is a plist file generated by an invocation of {@code actool} or + * {@link Optional#absent()} if no asset catalogues are present in this target and its + * dependencies. + * + * <p>All invocations of {@code actool} generate this kind of plist file, which contains metadata + * about the {@code app_icon} and {@code launch_image} if supplied. If neither an app icon or a + * launch image was supplied, the plist file generated is empty. + */ + private static Optional<Artifact> actoolPartialInfoplist( + RuleContext ruleContext, ObjcProvider objcProvider) { + if (objcProvider.hasAssetCatalogs()) { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + return Optional.of(intermediateArtifacts.actoolPartialInfoplist()); + } else { + return Optional.absent(); + } + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/BundleableFile.java b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleableFile.java new file mode 100644 index 0000000000..eea7bf06a0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleableFile.java @@ -0,0 +1,149 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ArtifactListAttribute.BUNDLE_IMPORTS; +import static com.google.devtools.build.lib.rules.objc.ObjcCommon.BUNDLE_CONTAINER_TYPE; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.BundleFile; +import com.google.devtools.build.xcode.util.Value; + +/** + * Represents a file which is processed to another file and bundled. It contains the + * {@code Artifact} corresponding to the original file as well as the {@code Artifact} for the file + * converted to its bundled form. Examples of files that fit this pattern are .strings and .xib + * files. + */ +public final class BundleableFile extends Value<BundleableFile> { + static final int EXECUTABLE_EXTERNAL_FILE_ATTRIBUTE = 0100755 << 16; + static final int DEFAULT_EXTERNAL_FILE_ATTRIBUTE = 0100644 << 16; + + private final Artifact bundled; + private final String bundlePath; + private final int zipExternalFileAttribute; + + /** + * Creates an instance whose {@code zipExternalFileAttribute} value is + * {@link #DEFAULT_EXTERNAL_FILE_ATTRIBUTE}. + */ + BundleableFile(Artifact bundled, String bundlePath) { + this(bundled, bundlePath, DEFAULT_EXTERNAL_FILE_ATTRIBUTE); + } + + /** + * @param bundled the {@link Artifact} whose data is placed in the bundle + * @param bundlePath the path of the file in the bundle + * @param the external file attribute of the file in the central directory of the bundle (zip + * file). The lower 16 bits contain the MS-DOS file attributes. The upper 16 bits contain the + * Unix file attributes, for instance 0100755 (octal) for a regular file with permissions + * {@code rwxr-xr-x}. + */ + BundleableFile(Artifact bundled, String bundlePath, int zipExternalFileAttribute) { + super(new ImmutableMap.Builder<String, Object>() + .put("bundled", bundled) + .put("bundlePath", bundlePath) + .put("zipExternalFileAttribute", zipExternalFileAttribute) + .build()); + this.bundled = bundled; + this.bundlePath = bundlePath; + this.zipExternalFileAttribute = zipExternalFileAttribute; + } + + static String bundlePath(PathFragment path) { + String containingDir = path.getParentDirectory().getBaseName(); + return (containingDir.endsWith(".lproj") ? (containingDir + "/") : "") + path.getBaseName(); + } + + /** + * Given a sequence of non-compiled resource files, returns a sequence of the same length of + * instances of this class. Non-compiled resource files are resources which are not processed + * before placing them in the final bundle. This is different from (for example) {@code .strings} + * and {@code .xib} files, which must be converted to binary plist form or compiled. + * + * @param files a sequence of artifacts corresponding to non-compiled resource files + */ + public static Iterable<BundleableFile> nonCompiledResourceFiles(Iterable<Artifact> files) { + ImmutableList.Builder<BundleableFile> result = new ImmutableList.Builder<>(); + for (Artifact file : files) { + result.add(new BundleableFile(file, bundlePath(file.getExecPath()))); + } + return result.build(); + } + + /** + * Returns an instance for every file in a bundle directory. + * <p> + * This uses the parent-most container matching {@code *.bundle} as the bundle root. + * TODO(bazel-team): add something like an import_root attribute to specify this explicitly, which + * will be helpful if a bundle that appears to be nested needs to be imported alone. + */ + public static Iterable<BundleableFile> bundleImportsFromRule(RuleContext context) { + ImmutableList.Builder<BundleableFile> result = new ImmutableList.Builder<>(); + for (Artifact artifact : BUNDLE_IMPORTS.get(context)) { + for (PathFragment container : + ObjcCommon.farthestContainerMatching(BUNDLE_CONTAINER_TYPE, artifact).asSet()) { + // TODO(bazel-team): Figure out if we need to remove symbols of architectures we aren't + // building for from the binary in the bundle. + result.add(new BundleableFile( + artifact, + // The path from the artifact's container (including the container), to the artifact + // itself. For instance, if artifact is foo/bar.bundle/baz, then this value + // is bar.bundle/baz. + artifact.getExecPath().relativeTo(container.getParentDirectory()).getSafePathString())); + } + } + return result.build(); + } + + /** + * The artifact that is ultimately bundled. + */ + public Artifact getBundled() { + return bundled; + } + + /** + * Returns bundle files for each given strings file. These are used to merge the strings files to + * the final application bundle. + */ + public static Iterable<BundleFile> toBundleFiles(Iterable<BundleableFile> files) { + ImmutableList.Builder<BundleFile> result = new ImmutableList.Builder<>(); + for (BundleableFile file : files) { + result.add(BundleFile.newBuilder() + .setBundlePath(file.bundlePath) + .setSourceFile(file.bundled.getExecPathString()) + .setExternalFileAttribute(file.zipExternalFileAttribute) + .build()); + } + return result.build(); + } + + /** + * Returns the artifacts for the bundled files. These can be used, for instance, as the input + * files to add to the bundlemerge action for a bundle that contains all the given files. + */ + public static Iterable<Artifact> toArtifacts(Iterable<BundleableFile> files) { + ImmutableList.Builder<Artifact> result = new ImmutableList.Builder<>(); + for (BundleableFile file : files) { + result.add(file.bundled); + } + return result.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/Bundling.java b/src/main/java/com/google/devtools/build/lib/rules/objc/Bundling.java new file mode 100644 index 0000000000..484c553678 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/Bundling.java @@ -0,0 +1,254 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.ASSET_CATALOG; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_FILE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.MERGE_ZIP; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.NESTED_BUNDLE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCDATAMODEL; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.xcode.util.Value; + +import java.util.Map; + +/** + * Contains information regarding the creation of an iOS bundle. + */ +@Immutable +final class Bundling extends Value<Bundling> { + static final class Builder { + private String name; + private String bundleDirSuffix; + private ImmutableList<BundleableFile> extraBundleFiles = ImmutableList.of(); + private ObjcProvider objcProvider; + private InfoplistMerging infoplistMerging; + private IntermediateArtifacts intermediateArtifacts; + + public Builder setName(String name) { + this.name = name; + return this; + } + + public Builder setBundleDirSuffix(String bundleDirSuffix) { + this.bundleDirSuffix = bundleDirSuffix; + return this; + } + + public Builder setExtraBundleFiles(ImmutableList<BundleableFile> extraBundleFiles) { + this.extraBundleFiles = extraBundleFiles; + return this; + } + + public Builder setObjcProvider(ObjcProvider objcProvider) { + this.objcProvider = objcProvider; + return this; + } + + public Builder setInfoplistMerging(InfoplistMerging infoplistMerging) { + this.infoplistMerging = infoplistMerging; + return this; + } + + public Builder setIntermediateArtifacts(IntermediateArtifacts intermediateArtifacts) { + this.intermediateArtifacts = intermediateArtifacts; + return this; + } + + private static NestedSet<Artifact> nestedBundleContentArtifacts(Iterable<Bundling> bundles) { + NestedSetBuilder<Artifact> artifacts = NestedSetBuilder.stableOrder(); + for (Bundling bundle : bundles) { + artifacts.addTransitive(bundle.getBundleContentArtifacts()); + } + return artifacts.build(); + } + + public Bundling build() { + Preconditions.checkNotNull(intermediateArtifacts, "intermediateArtifacts"); + + Optional<Artifact> actoolzipOutput = Optional.absent(); + if (!Iterables.isEmpty(objcProvider.get(ASSET_CATALOG))) { + actoolzipOutput = Optional.of(intermediateArtifacts.actoolzipOutput()); + } + + Optional<Artifact> combinedArchitectureBinary = Optional.absent(); + if (!Iterables.isEmpty(objcProvider.get(LIBRARY)) + || !Iterables.isEmpty(objcProvider.get(IMPORTED_LIBRARY))) { + combinedArchitectureBinary = + Optional.of(intermediateArtifacts.combinedArchitectureBinary(bundleDirSuffix)); + } + + NestedSet<Artifact> mergeZips = NestedSetBuilder.<Artifact>stableOrder() + .addAll(actoolzipOutput.asSet()) + .addTransitive(objcProvider.get(MERGE_ZIP)) + .build(); + NestedSet<Artifact> bundleContentArtifacts = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(nestedBundleContentArtifacts(objcProvider.get(NESTED_BUNDLE))) + .addAll(combinedArchitectureBinary.asSet()) + .addAll(infoplistMerging.getPlistWithEverything().asSet()) + .addTransitive(mergeZips) + .addAll(BundleableFile.toArtifacts(extraBundleFiles)) + .addAll(BundleableFile.toArtifacts(objcProvider.get(BUNDLE_FILE))) + .addAll(Xcdatamodel.outputZips(objcProvider.get(XCDATAMODEL))) + .build(); + + return new Bundling(name, bundleDirSuffix, combinedArchitectureBinary, extraBundleFiles, + objcProvider, infoplistMerging, actoolzipOutput, bundleContentArtifacts, mergeZips); + } + } + + private final String name; + private final String bundleDirSuffix; + private final Optional<Artifact> combinedArchitectureBinary; + private final ImmutableList<BundleableFile> extraBundleFiles; + private final ObjcProvider objcProvider; + private final InfoplistMerging infoplistMerging; + private final Optional<Artifact> actoolzipOutput; + private final NestedSet<Artifact> bundleContentArtifacts; + private final NestedSet<Artifact> mergeZips; + + private Bundling( + String name, + String bundleDirSuffix, + Optional<Artifact> combinedArchitectureBinary, + ImmutableList<BundleableFile> extraBundleFiles, + ObjcProvider objcProvider, + InfoplistMerging infoplistMerging, + Optional<Artifact> actoolzipOutput, + NestedSet<Artifact> bundleContentArtifacts, + NestedSet<Artifact> mergeZips) { + super(new ImmutableMap.Builder<String, Object>() + .put("name", name) + .put("bundleDirSuffix", bundleDirSuffix) + .put("combinedArchitectureBinary", combinedArchitectureBinary) + .put("extraBundleFiles", extraBundleFiles) + .put("objcProvider", objcProvider) + .put("infoplistMerging", infoplistMerging) + .put("actoolzipOutput", actoolzipOutput) + .put("bundleContentArtifacts", bundleContentArtifacts) + .put("mergeZips", mergeZips) + .build()); + this.name = name; + this.bundleDirSuffix = bundleDirSuffix; + this.combinedArchitectureBinary = combinedArchitectureBinary; + this.extraBundleFiles = extraBundleFiles; + this.objcProvider = objcProvider; + this.infoplistMerging = infoplistMerging; + this.actoolzipOutput = actoolzipOutput; + this.bundleContentArtifacts = bundleContentArtifacts; + this.mergeZips = mergeZips; + } + + /** + * The bundle directory. For apps, {@code "Payload/" + bundleDir} is the directory in the bundle + * zip archive in which every file is found including the linked binary, nested bundles, and + * everything returned by {@link #getExtraBundleFiles()}. In an application bundle, for instance, + * this function returns {@code "(name).app"}. + */ + public String getBundleDir() { + return name + bundleDirSuffix; + } + + /** + * The name of the bundle, from which the bundle root and the path of the linked binary in the + * bundle archive are derived. + */ + public String getName() { + return name; + } + + /** + * An {@link Optional} with the linked binary artifact, or {@link Optional#absent()} if it is + * empty and should not be included in the bundle. + */ + public Optional<Artifact> getCombinedArchitectureBinary() { + return combinedArchitectureBinary; + } + + /** + * Extra bundle files to include in the bundle which are not automatically deduced by the contents + * of the provider. These files are placed under the bundle root (possibly nested, of course, + * depending on the bundle path of the files). + */ + public ImmutableList<BundleableFile> getExtraBundleFiles() { + return extraBundleFiles; + } + + /** + * The {@link ObjcProvider} for this bundle. + */ + public ObjcProvider getObjcProvider() { + return objcProvider; + } + + /** + * Information on the Info.plist and its merge inputs for this bundle. Note that an infoplist is + * only included in the bundle if it has one or more merge inputs. + */ + public InfoplistMerging getInfoplistMerging() { + return infoplistMerging; + } + + /** + * The location of the actoolzip output for this bundle. This is non-absent only included in the + * bundle if there is at least one asset catalog artifact supplied by + * {@link ObjcProvider#ASSET_CATALOG}. + */ + public Optional<Artifact> getActoolzipOutput() { + return actoolzipOutput; + } + + /** + * Returns all zip files whose contents should be merged into this bundle under the main bundle + * directory. For instance, if a merge zip contains files a/b and c/d, then the resulting bundling + * would have additional files at: + * <ul> + * <li>{bundleDir}/a/b + * <li>{bundleDir}/c/d + * </ul> + */ + public NestedSet<Artifact> getMergeZips() { + return mergeZips; + } + + /** + * Returns the variable substitutions that should be used when merging the plist info file of + * this bundle. + */ + public Map<String, String> variableSubstitutions() { + return ImmutableMap.of( + "EXECUTABLE_NAME", name, + "BUNDLE_NAME", name + bundleDirSuffix, + "PRODUCT_NAME", name); + } + + /** + * Returns the artifacts that are required to generate this bundle. + */ + public NestedSet<Artifact> getBundleContentArtifacts() { + return bundleContentArtifacts; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationArtifacts.java b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationArtifacts.java new file mode 100644 index 0000000000..5aac139e57 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationArtifacts.java @@ -0,0 +1,98 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; + +/** + * Artifacts related to compilation. Any rule containing compilable sources will create an instance + * of this class. + */ +final class CompilationArtifacts { + static class Builder { + private Iterable<Artifact> srcs = ImmutableList.of(); + private Iterable<Artifact> nonArcSrcs = ImmutableList.of(); + private Optional<Artifact> pchFile; + private IntermediateArtifacts intermediateArtifacts; + + Builder addSrcs(Iterable<Artifact> srcs) { + this.srcs = Iterables.concat(this.srcs, srcs); + return this; + } + + Builder addNonArcSrcs(Iterable<Artifact> nonArcSrcs) { + this.nonArcSrcs = Iterables.concat(this.nonArcSrcs, nonArcSrcs); + return this; + } + + Builder setPchFile(Optional<Artifact> pchFile) { + Preconditions.checkState(this.pchFile == null, + "pchFile is already set to: %s", this.pchFile); + this.pchFile = Preconditions.checkNotNull(pchFile); + return this; + } + + Builder setIntermediateArtifacts(IntermediateArtifacts intermediateArtifacts) { + Preconditions.checkState(this.intermediateArtifacts == null, + "intermediateArtifacts is already set to: %s", this.intermediateArtifacts); + this.intermediateArtifacts = intermediateArtifacts; + return this; + } + + CompilationArtifacts build() { + Optional<Artifact> archive = Optional.absent(); + if (!Iterables.isEmpty(srcs) || !Iterables.isEmpty(nonArcSrcs)) { + archive = Optional.of(intermediateArtifacts.archive()); + } + return new CompilationArtifacts(srcs, nonArcSrcs, archive, pchFile); + } + } + + private final Iterable<Artifact> srcs; + private final Iterable<Artifact> nonArcSrcs; + private final Optional<Artifact> archive; + private final Optional<Artifact> pchFile; + + private CompilationArtifacts( + Iterable<Artifact> srcs, + Iterable<Artifact> nonArcSrcs, + Optional<Artifact> archive, + Optional<Artifact> pchFile) { + this.srcs = Preconditions.checkNotNull(srcs); + this.nonArcSrcs = Preconditions.checkNotNull(nonArcSrcs); + this.archive = Preconditions.checkNotNull(archive); + this.pchFile = Preconditions.checkNotNull(pchFile); + } + + public Iterable<Artifact> getSrcs() { + return srcs; + } + + public Iterable<Artifact> getNonArcSrcs() { + return nonArcSrcs; + } + + public Optional<Artifact> getArchive() { + return archive; + } + + public Optional<Artifact> getPchFile() { + return pchFile; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java new file mode 100644 index 0000000000..1e23798363 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java @@ -0,0 +1,235 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.NON_ARC_SRCS_TYPE; +import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.SRCS_TYPE; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkArgs; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkInputs; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.CompilationAttributes; +import com.google.devtools.build.lib.rules.objc.XcodeProvider.Builder; +import com.google.devtools.build.lib.shell.ShellUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Support for rules that compile sources. Provides ways to determine files that should be output, + * registering Xcode settings and generating the various actions that might be needed for + * compilation. + * + * <p>Methods on this class can be called in any order without impacting the result. + */ +final class CompilationSupport { + + @VisibleForTesting + static final String ABSOLUTE_INCLUDES_PATH_FORMAT = + "The path '%s' is absolute, but only relative paths are allowed."; + + /** + * Returns information about the given rule's compilation artifacts. + */ + // TODO(bazel-team): Remove this information from ObjcCommon and move it internal to this class. + static CompilationArtifacts compilationArtifacts(RuleContext ruleContext) { + return new CompilationArtifacts.Builder() + .addSrcs(ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET) + .errorsForNonMatching(SRCS_TYPE) + .list()) + .addNonArcSrcs(ruleContext.getPrerequisiteArtifacts("non_arc_srcs", Mode.TARGET) + .errorsForNonMatching(NON_ARC_SRCS_TYPE) + .list()) + .setIntermediateArtifacts(ObjcRuleClasses.intermediateArtifacts(ruleContext)) + .setPchFile(Optional.fromNullable(ruleContext.getPrerequisiteArtifact("pch", Mode.TARGET))) + .build(); + } + + private final RuleContext ruleContext; + private final CompilationAttributes attributes; + + /** + * Creates a new compilation support for the given rule. + */ + CompilationSupport(RuleContext ruleContext) { + this.ruleContext = ruleContext; + this.attributes = new CompilationAttributes(ruleContext); + } + + /** + * Registers all actions necessary to compile this rule's sources and archive them. + * + * @param common common information about this rule and its dependencies + * @param optionsProvider option and plist information about this rule and its dependencies + * + * @return this compilation support + */ + CompilationSupport registerCompileAndArchiveActions( + ObjcCommon common, OptionsProvider optionsProvider) { + if (common.getCompilationArtifacts().isPresent()) { + ObjcRuleClasses.actionsBuilder(ruleContext).registerCompileAndArchiveActions( + common.getCompilationArtifacts().get(), common.getObjcProvider(), optionsProvider); + } + return this; + } + + /** + * Registers any actions necessary to link this rule and its dependencies. Debug symbols are + * generated if {@link ObjcConfiguration#generateDebugSymbols()} is set. + * + * @param objcProvider common information about this rule's attributes and its dependencies + * @param extraLinkArgs any additional arguments to pass to the linker + * @param extraLinkInputs any additional input artifacts to pass to the link action + * + * @return this compilation support + */ + CompilationSupport registerLinkActions(ObjcProvider objcProvider, ExtraLinkArgs extraLinkArgs, + ExtraLinkInputs extraLinkInputs) { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + Optional<Artifact> dsymBundle; + if (ObjcRuleClasses.objcConfiguration(ruleContext).generateDebugSymbols()) { + registerDsymActions(); + dsymBundle = Optional.of(intermediateArtifacts.dsymBundle()); + } else { + dsymBundle = Optional.absent(); + } + + ObjcRuleClasses.actionsBuilder(ruleContext).registerLinkAction( + intermediateArtifacts.singleArchitectureBinary(), objcProvider, extraLinkArgs, + extraLinkInputs, dsymBundle); + return this; + } + + /** + * Registers actions that compile and archive j2Objc dependencies of this rule. + * + * @param optionsProvider option and plist information about this rule and its dependencies + * @param objcProvider common information about this rule's attributes and its dependencies + * + * @return this compilation support + */ + CompilationSupport registerJ2ObjcCompileAndArchiveActions( + OptionsProvider optionsProvider, ObjcProvider objcProvider) { + for (J2ObjcSource j2ObjcSource : ObjcRuleClasses.j2ObjcSrcsProvider(ruleContext).getSrcs()) { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.j2objcIntermediateArtifacts(ruleContext, j2ObjcSource); + CompilationArtifacts compilationArtifact = new CompilationArtifacts.Builder() + .addNonArcSrcs(j2ObjcSource.getObjcSrcs()) + .setIntermediateArtifacts(intermediateArtifacts) + .setPchFile(Optional.<Artifact>absent()) + .build(); + ObjcActionsBuilder actionBuilder = new ObjcActionsBuilder( + ruleContext, + intermediateArtifacts, + ObjcRuleClasses.objcConfiguration(ruleContext), + ruleContext.getConfiguration(), + ruleContext); + actionBuilder + .registerCompileAndArchiveActions(compilationArtifact, objcProvider, optionsProvider); + } + + return this; + } + + /** + * Sets compilation-related Xcode project information on the given provider builder. + * + * @param common common information about this rule's attributes and its dependencies + * @param optionsProvider option and plist information about this rule and its dependencies + * @return this compilation support + */ + CompilationSupport addXcodeSettings(Builder xcodeProviderBuilder, + ObjcCommon common, OptionsProvider optionsProvider) { + ObjcConfiguration objcConfiguration = ObjcRuleClasses.objcConfiguration(ruleContext); + for (CompilationArtifacts artifacts : common.getCompilationArtifacts().asSet()) { + xcodeProviderBuilder.setCompilationArtifacts(artifacts); + } + xcodeProviderBuilder + .addHeaders(attributes.hdrs()) + .addUserHeaderSearchPaths(ObjcCommon.userHeaderSearchPaths(ruleContext.getConfiguration())) + .addHeaderSearchPaths("$(WORKSPACE_ROOT)", attributes.headerSearchPaths()) + .addHeaderSearchPaths("$(SDKROOT)/usr/include", attributes.sdkIncludes()) + .addCompilationModeCopts(objcConfiguration.getCoptsForCompilationMode()) + .addCopts(objcConfiguration.getCopts()) + .addCopts(optionsProvider.getCopts()); + return this; + } + + /** + * Validates compilation-related attributes on this rule. + * + * @return this compilation support + */ + CompilationSupport validateAttributes() { + for (PathFragment absoluteInclude : + Iterables.filter(attributes.includes(), PathFragment.IS_ABSOLUTE)) { + ruleContext.attributeError( + "includes", String.format(ABSOLUTE_INCLUDES_PATH_FORMAT, absoluteInclude)); + } + + return this; + } + + private CompilationSupport registerDsymActions() { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + Artifact dsymBundle = intermediateArtifacts.dsymBundle(); + Artifact debugSymbolFile = intermediateArtifacts.dsymSymbol(); + ruleContext.registerAction(new SpawnAction.Builder() + .setMnemonic("UnzipDsym") + .setProgressMessage("Unzipping dSYM file: " + ruleContext.getLabel()) + .setExecutable(new PathFragment("/usr/bin/unzip")) + .addInput(dsymBundle) + .setCommandLine(CustomCommandLine.builder() + .add(dsymBundle.getExecPathString()) + .add("-d") + .add(stripSuffix(dsymBundle.getExecPathString(), + IntermediateArtifacts.TMP_DSYM_BUNDLE_SUFFIX) + ".app.dSYM") + .build()) + .addOutput(intermediateArtifacts.dsymPlist()) + .addOutput(debugSymbolFile) + .build(ruleContext)); + + Artifact dumpsyms = ruleContext.getPrerequisiteArtifact("$dumpsyms", Mode.HOST); + Artifact breakpadFile = intermediateArtifacts.breakpadSym(); + ruleContext.registerAction(new SpawnAction.Builder() + .setMnemonic("GenBreakpad") + .setProgressMessage("Generating breakpad file: " + ruleContext.getLabel()) + .setShellCommand(ImmutableList.of("/bin/bash", "-c")) + .setExecutionInfo(ImmutableMap.of(ExecutionRequirements.REQUIRES_DARWIN, "")) + .addInput(dumpsyms) + .addInput(debugSymbolFile) + .addArgument(String.format("%s %s > %s", + ShellUtils.shellEscape(dumpsyms.getExecPathString()), + ShellUtils.shellEscape(debugSymbolFile.getExecPathString()), + ShellUtils.shellEscape(breakpadFile.getExecPathString()))) + .addOutput(breakpadFile) + .build(ruleContext)); + return this; + } + + private String stripSuffix(String str, String suffix) { + // TODO(bazel-team): Throw instead of returning null? + return str.endsWith(suffix) ? str.substring(0, str.length() - suffix.length()) : null; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/CompiledResourceFile.java b/src/main/java/com/google/devtools/build/lib/rules/objc/CompiledResourceFile.java new file mode 100644 index 0000000000..cd84710c70 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/CompiledResourceFile.java @@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; + +/** + * Represents a strings file. + */ +public class CompiledResourceFile { + private final Artifact original; + private final BundleableFile bundled; + + private CompiledResourceFile(Artifact original, BundleableFile bundled) { + this.original = Preconditions.checkNotNull(original); + this.bundled = Preconditions.checkNotNull(bundled); + } + + /** + * The checked-in version of the bundled file. + */ + public Artifact getOriginal() { + return original; + } + + public BundleableFile getBundled() { + return bundled; + } + + public static final Function<CompiledResourceFile, BundleableFile> TO_BUNDLED = + new Function<CompiledResourceFile, BundleableFile>() { + @Override + public BundleableFile apply(CompiledResourceFile input) { + return input.bundled; + } + }; + + /** + * Given a sequence of artifacts corresponding to {@code .strings} files, returns a sequence of + * the same length of instances of this class. The value returned by {@link #getBundled()} of each + * instance will be the plist file in binary form. + */ + public static Iterable<CompiledResourceFile> fromStringsFiles( + IntermediateArtifacts intermediateArtifacts, Iterable<Artifact> strings) { + ImmutableList.Builder<CompiledResourceFile> result = new ImmutableList.Builder<>(); + for (Artifact originalFile : strings) { + Artifact binaryFile = intermediateArtifacts.convertedStringsFile(originalFile); + result.add(new CompiledResourceFile( + originalFile, + new BundleableFile(binaryFile, BundleableFile.bundlePath(originalFile.getExecPath())))); + } + return result.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ExecutionRequirements.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ExecutionRequirements.java new file mode 100644 index 0000000000..22571368be --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ExecutionRequirements.java @@ -0,0 +1,23 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +/** + * Strings used to express requirements on action execution environments. + */ +public class ExecutionRequirements { + /** If an action would not successfully run other than on Darwin. */ + public static final String REQUIRES_DARWIN = "requires-darwin"; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/InfoplistMerging.java b/src/main/java/com/google/devtools/build/lib/rules/objc/InfoplistMerging.java new file mode 100644 index 0000000000..58369ada51 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/InfoplistMerging.java @@ -0,0 +1,142 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.xcode.util.Interspersing; + +/** + * Supplies information regarding Infoplist merging for a particular binary. This includes: + * <ul> + * <li>the Info.plist which contains the fields from every source. If there is only one source + * plist, this is that plist. + * <li>the action to merge all the Infoplists into a single one. This is present even if there is + * only one Infoplist, to prevent a Bazel error when an Artifact does not have a generating + * action. + * </ul> + */ +class InfoplistMerging { + static class Builder { + private final ActionConstructionContext context; + private NestedSet<Artifact> inputPlists; + private FilesToRunProvider plmerge; + private IntermediateArtifacts intermediateArtifacts; + + public Builder(ActionConstructionContext context) { + this.context = Preconditions.checkNotNull(context); + } + + public Builder setInputPlists(NestedSet<Artifact> inputPlists) { + Preconditions.checkState(this.inputPlists == null); + this.inputPlists = inputPlists; + return this; + } + + public Builder setPlmerge(FilesToRunProvider plmerge) { + Preconditions.checkState(this.plmerge == null); + this.plmerge = plmerge; + return this; + } + + public Builder setIntermediateArtifacts(IntermediateArtifacts intermediateArtifacts) { + this.intermediateArtifacts = intermediateArtifacts; + return this; + } + + /** + * This static factory method prevents retention of the outer {@link Builder} class reference by + * the anonymous {@link CommandLine} instance. + */ + private static CommandLine mergeCommandLine( + final NestedSet<Artifact> inputPlists, final Artifact mergedInfoplist) { + return new CommandLine() { + @Override + public Iterable<String> arguments() { + return new ImmutableList.Builder<String>() + .addAll(Interspersing.beforeEach( + "--source_file", Artifact.toExecPaths(inputPlists))) + .add("--out_file", mergedInfoplist.getExecPathString()) + .build(); + } + }; + } + + public InfoplistMerging build() { + Preconditions.checkNotNull(intermediateArtifacts, "intermediateArtifacts"); + + Optional<Artifact> plistWithEverything = Optional.absent(); + Action[] mergeActions = new Action[0]; + + int inputs = Iterables.size(inputPlists); + if (inputs == 1) { + plistWithEverything = Optional.of(Iterables.getOnlyElement(inputPlists)); + } else if (inputs > 1) { + Artifact merged = intermediateArtifacts.mergedInfoplist(); + + plistWithEverything = Optional.of(merged); + mergeActions = new SpawnAction.Builder() + .setMnemonic("MergeInfoPlistFiles") + .setExecutable(plmerge) + .setCommandLine(mergeCommandLine(inputPlists, merged)) + .addTransitiveInputs(inputPlists) + .addOutput(merged) + .build(context); + } + + return new InfoplistMerging(plistWithEverything, mergeActions, inputPlists); + } + } + + private final Optional<Artifact> plistWithEverything; + private final Action[] mergeActions; + private final NestedSet<Artifact> inputPlists; + + private InfoplistMerging(Optional<Artifact> plistWithEverything, Action[] mergeActions, + NestedSet<Artifact> inputPlists) { + this.plistWithEverything = plistWithEverything; + this.mergeActions = mergeActions; + this.inputPlists = inputPlists; + } + + /** + * Creates action to merge multiple Info.plist files of a binary into a single Info.plist. No + * action is necessary if there is only one source. + */ + public Action[] getMergeAction() { + return mergeActions; + } + + /** + * An {@link Optional} with the merged infoplist, or {@link Optional#absent()} if there are no + * merge inputs and it should not be included in the bundle. + */ + public Optional<Artifact> getPlistWithEverything() { + return plistWithEverything; + } + + public NestedSet<Artifact> getInputPlists() { + return inputPlists; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IntermediateArtifacts.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IntermediateArtifacts.java new file mode 100644 index 0000000000..b84bf0336b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IntermediateArtifacts.java @@ -0,0 +1,220 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Factory class for generating artifacts which are used as intermediate output. + */ +// TODO(bazel-team): This should really be named DerivedArtifacts as it contains methods for +// final as well as intermediate artifacts. +final class IntermediateArtifacts { + + /** + * Extension used on the temporary dsym bundle location. Must end in {@code .dSYM} for dsymutil + * to generate a plist file. + */ + static final String TMP_DSYM_BUNDLE_SUFFIX = ".temp.app.dSYM"; + + private final AnalysisEnvironment analysisEnvironment; + private final Root binDirectory; + private final Label ownerLabel; + private final String archiveFileNameSuffix; + + IntermediateArtifacts( + AnalysisEnvironment analysisEnvironment, Root binDirectory, Label ownerLabel, + String archiveFileNameSuffix) { + this.analysisEnvironment = Preconditions.checkNotNull(analysisEnvironment); + this.binDirectory = Preconditions.checkNotNull(binDirectory); + this.ownerLabel = Preconditions.checkNotNull(ownerLabel); + this.archiveFileNameSuffix = Preconditions.checkNotNull(archiveFileNameSuffix); + } + + /** + * Returns a derived artifact in the bin directory obtained by appending some extension to the end + * of the given {@link PathFragment}. + */ + private Artifact appendExtension(PathFragment original, String extension) { + return analysisEnvironment.getDerivedArtifact( + FileSystemUtils.appendExtension(original, extension), binDirectory); + } + + /** + * Returns a derived artifact in the bin directory obtained by appending some extension to the end + * of the {@link PathFragment} corresponding to the owner {@link Label}. + */ + private Artifact appendExtension(String extension) { + return appendExtension(ownerLabel.toPathFragment(), extension); + } + + /** + * The output of using {@code actooloribtoolzip} to run {@code actool} for a given bundle which is + * merged under the {@code .app} or {@code .bundle} directory root. + */ + public Artifact actoolzipOutput() { + return appendExtension(".actool.zip"); + } + + /** + * Output of the partial infoplist generated by {@code actool} when given the + * {@code --output-partial-info-plist [path]} flag. + */ + public Artifact actoolPartialInfoplist() { + return appendExtension(".actool-PartialInfo.plist"); + } + + /** + * The Info.plist file for a bundle which is comprised of more than one originating plist file. + * This is not needed for a bundle which has no source Info.plist files, or only one Info.plist + * file, since no merging occurs in that case. + */ + public Artifact mergedInfoplist() { + return appendExtension("-MergedInfo.plist"); + } + + /** + * The .objlist file, which contains a list of paths of object files to archive and is read by + * libtool in the archive action. + */ + public Artifact objList() { + return appendExtension(".objlist"); + } + + /** + * The artifact which is the binary (or library) which is comprised of one or more .a files linked + * together. + */ + public Artifact singleArchitectureBinary() { + return appendExtension("_bin"); + } + + /** + * Lipo binary generated by combining one or more linked binaries. This binary is the one included + * in generated bundles and invoked as entry point to the application. + * + * @param bundleDirSuffix suffix of the bundle containing this binary + */ + public Artifact combinedArchitectureBinary(String bundleDirSuffix) { + String baseName = ownerLabel.toPathFragment().getBaseName(); + return appendExtension(bundleDirSuffix + "/" + baseName); + } + + /** + * The {@code .a} file which contains all the compiled sources for a rule. + */ + public Artifact archive() { + PathFragment labelPath = ownerLabel.toPathFragment(); + PathFragment rootRelative = labelPath + .getParentDirectory() + .getRelative(String.format("lib%s%s.a", labelPath.getBaseName(), archiveFileNameSuffix)); + return analysisEnvironment.getDerivedArtifact(rootRelative, binDirectory); + } + + /** + * The debug symbol bundle file which contains debug symbols generated by dsymutil. + */ + public Artifact dsymBundle() { + return appendExtension(TMP_DSYM_BUNDLE_SUFFIX); + } + + /** + * The artifact for the .o file that should be generated when compiling the {@code source} + * artifact. + */ + public Artifact objFile(Artifact source) { + return analysisEnvironment.getDerivedArtifact( + FileSystemUtils.replaceExtension( + AnalysisUtils.getUniqueDirectory(ownerLabel, new PathFragment("_objs")) + .getRelative(source.getRootRelativePath()), + ".o"), + binDirectory); + } + + /** + * Returns the artifact corresponding to the pbxproj control file, which specifies the information + * required to generate the Xcode project file. + */ + public Artifact pbxprojControlArtifact() { + return appendExtension(".xcodeproj-control"); + } + + /** + * The artifact which contains the zipped-up results of compiling the storyboard. This is merged + * into the final bundle under the {@code .app} or {@code .bundle} directory root. + */ + public Artifact compiledStoryboardZip(Artifact input) { + return appendExtension("/" + BundleableFile.bundlePath(input.getExecPath()) + ".zip"); + } + + /** + * Returns the artifact which is the output of building an entire xcdatamodel[d] made of artifacts + * specified by a single rule. + * + * @param containerDir the containing *.xcdatamodeld or *.xcdatamodel directory + * @return the artifact for the zipped up compilation results. + */ + public Artifact compiledMomZipArtifact(PathFragment containerDir) { + return appendExtension( + "/" + FileSystemUtils.replaceExtension(containerDir, ".zip").getBaseName()); + } + + /** + * Returns the compiled (i.e. converted to binary plist format) artifact corresponding to the + * given {@code .strings} file. + */ + public Artifact convertedStringsFile(Artifact originalFile) { + return appendExtension(originalFile.getExecPath(), ".binary"); + } + + /** + * Returns the artifact corresponding to the zipped-up compiled form of the given {@code .xib} + * file. + */ + public Artifact compiledXibFileZip(Artifact originalFile) { + return analysisEnvironment.getDerivedArtifact( + FileSystemUtils.replaceExtension(originalFile.getExecPath(), ".nib.zip"), + binDirectory); + } + + /** + * Debug symbol plist generated for a linked binary. + */ + public Artifact dsymPlist() { + return appendExtension(".app.dSYM/Contents/Info.plist"); + } + + /** + * Debug symbol file generated for a linked binary. + */ + public Artifact dsymSymbol() { + return appendExtension( + String.format(".app.dSYM/Contents/Resources/DWARF/%s_bin", ownerLabel.getName())); + } + + /** + * Breakpad debug symbol representation. + */ + public Artifact breakpadSym() { + return appendExtension(".breakpad"); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosApplicationRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosApplicationRule.java new file mode 100644 index 0000000000..b81d9b8e80 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosApplicationRule.java @@ -0,0 +1,117 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.STRING; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.xcode.common.TargetDeviceFamily; + +/** + * Rule definition for ios_application. + */ +@BlazeRule(name = "$ios_application", + ancestors = { BaseRuleClasses.BaseRule.class, + ObjcRuleClasses.ObjcBaseResourcesRule.class, + ObjcRuleClasses.ObjcHasInfoplistRule.class, + ObjcRuleClasses.ObjcHasEntitlementsRule.class }, + type = RuleClassType.ABSTRACT) // TODO(bazel-team): Add factory once this becomes a real rule. +public class IosApplicationRule implements RuleDefinition { + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(app_icon) --> + The name of the application icon, which should be in one of the asset + catalogs of this target or a (transitive) dependency. In a new project, + this is initialized to "AppIcon" by Xcode. + <p> + If the application icon is not in an asset catalog, do not use this + attribute. Instead, add a CFBundleIcons entry to the Info.plist file. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("app_icon", STRING)) + /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(launch_image) --> + The name of the launch image, which should be in one of the asset + catalogs of this target or a (transitive) dependency. In a new project, + this is initialized to "LaunchImage" by Xcode. + <p> + If the launch image is not in an asset catalog, do not use this + attribute. Instead, add an appropriately-named image resource to the + bundle. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("launch_image", STRING)) + /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(bundle_id) --> + The bundle ID (reverse-DNS path followed by app name) of the binary. If none is specified, a + junk value will be used. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("bundle_id", STRING) + .value(new Attribute.ComputedDefault() { + @Override + public Object getDefault(AttributeMap rule) { + // For tests and similar, we don't want to force people to explicitly specify + // throw-away data. + return "example." + rule.getName(); + } + })) + /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(families) --> + The device families to which this binary is targeted. This is known as + the <code>TARGETED_DEVICE_FAMILY</code> build setting in Xcode project + files. It is a list of one or more of the strings <code>"iphone"</code> + and <code>"ipad"</code>. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("families", STRING_LIST) + .value(ImmutableList.of(TargetDeviceFamily.IPHONE.getNameInRule()))) + /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(provisioning_profile) --> + The provisioning profile (.mobileprovision file) to use when bundling + the application. + <p> + This is only used for non-simulator builds. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("provisioning_profile", LABEL) + .value(env.getLabel("//tools/objc:default_provisioning_profile")) + .allowedFileTypes(FileType.of(".mobileprovision"))) + // TODO(bazel-team): Consider ways to trim dependencies so that changes to deps of these + // tools don't trigger all objc_* targets. Right now we check-in deploy jars, but we + // need a less painful and error-prone way. + /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(binary) --> + The binary target included in the final bundle. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("binary", LABEL) + .allowedRuleClasses("objc_binary") + .allowedFileTypes() + .mandatory() + .direct_compile_time_input()) + .add(attr("$bundlemerge", LABEL).cfg(HOST).exec() + .value(env.getLabel("//tools/objc:bundlemerge"))) + .add(attr("$dumpsyms", LABEL).cfg(HOST).exec() + .value(env.getLabel("//tools/objc:dump_syms"))) + .build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosDevice.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDevice.java new file mode 100644 index 0000000000..2f01633a30 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDevice.java @@ -0,0 +1,42 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * Implementation for the "ios_device" rule. + */ +public final class IosDevice implements RuleConfiguredTargetFactory { + @Override + public ConfiguredTarget create(RuleContext context) throws InterruptedException { + IosDeviceProvider provider = new IosDeviceProvider.Builder() + .setType(context.attributes().get("type", STRING)) + .setIosVersion(context.attributes().get("ios_version", STRING)) + .setLocale(context.attributes().get("locale", STRING)) + .build(); + + return new RuleConfiguredTargetBuilder(context) + .add(RunfilesProvider.class, RunfilesProvider.EMPTY) + .add(IosDeviceProvider.class, provider) + .build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceProvider.java new file mode 100644 index 0000000000..6f01e11fbc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceProvider.java @@ -0,0 +1,73 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Provider that describes a simulator device. + */ +@Immutable +public final class IosDeviceProvider implements TransitiveInfoProvider { + /** A builder of {@link IosDeviceProvider}s. */ + public static final class Builder { + private String type; + private String iosVersion; + private String locale; + + public Builder setType(String type) { + this.type = type; + return this; + } + + public Builder setIosVersion(String iosVersion) { + this.iosVersion = iosVersion; + return this; + } + + public Builder setLocale(String locale) { + this.locale = locale; + return this; + } + + public IosDeviceProvider build() { + return new IosDeviceProvider(this); + } + } + + private final String type; + private final String iosVersion; + private final String locale; + + private IosDeviceProvider(Builder builder) { + this.type = Preconditions.checkNotNull(builder.type); + this.iosVersion = Preconditions.checkNotNull(builder.iosVersion); + this.locale = Preconditions.checkNotNull(builder.locale); + } + + public String getType() { + return type; + } + + public String getIosVersion() { + return iosVersion; + } + + public String getLocale() { + return locale; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceRule.java new file mode 100644 index 0000000000..e028e700a4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceRule.java @@ -0,0 +1,67 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for ios_device. + */ +@BlazeRule(name = "ios_device", + factoryClass = IosDevice.class, + ancestors = { BaseRuleClasses.RuleBase.class }) +public final class IosDeviceRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE(ios_device).ATTRIBUTE(ios_version) --> + The operating system version of the device. This corresponds to the + <code>simctl</code> runtime. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("ios_version", STRING) + .mandatory()) + /* <!-- #BLAZE_RULE(ios_device).ATTRIBUTE(type) --> + The hardware type. This corresponds to the <code>simctl</code> device + type. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("type", STRING) + .mandatory()) + .add(attr("locale", STRING) + .undocumented("this is not yet supported by any test runner") + .value("en")) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = ios_device, TYPE = BINARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule defines an iOS device profile that defines a simulator against +which to run tests</p>. + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/ diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosSdkCommands.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosSdkCommands.java new file mode 100644 index 0000000000..0a9fb7b6a6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosSdkCommands.java @@ -0,0 +1,155 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_DIR; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.common.Platform; +import com.google.devtools.build.xcode.util.Interspersing; +import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.XcodeprojBuildSetting; + +import java.util.List; + +/** + * Utility code for use when generating iOS SDK commands. + */ +public class IosSdkCommands { + public static final String DEVELOPER_DIR = "/Applications/Xcode.app/Contents/Developer"; + public static final String BIN_DIR = + DEVELOPER_DIR + "/Toolchains/XcodeDefault.xctoolchain/usr/bin"; + public static final String ACTOOL_PATH = DEVELOPER_DIR + "/usr/bin/actool"; + public static final String IBTOOL_PATH = DEVELOPER_DIR + "/usr/bin/ibtool"; + public static final String MOMC_PATH = DEVELOPER_DIR + "/usr/bin/momc"; + + // There is a handy reference to many clang warning flags at + // http://nshipster.com/clang-diagnostics/ + // There is also a useful narrative for many Xcode settings at + // http://www.xs-labs.com/en/blog/2011/02/04/xcode-build-settings/ + @VisibleForTesting + static final ImmutableMap<String, String> DEFAULT_WARNINGS = + new ImmutableMap.Builder<String, String>() + .put("GCC_WARN_64_TO_32_BIT_CONVERSION", "-Wshorten-64-to-32") + .put("CLANG_WARN_BOOL_CONVERSION", "-Wbool-conversion") + .put("CLANG_WARN_CONSTANT_CONVERSION", "-Wconstant-conversion") + // Double-underscores are intentional - thanks Xcode. + .put("CLANG_WARN__DUPLICATE_METHOD_MATCH", "-Wduplicate-method-match") + .put("CLANG_WARN_EMPTY_BODY", "-Wempty-body") + .put("CLANG_WARN_ENUM_CONVERSION", "-Wenum-conversion") + .put("CLANG_WARN_INT_CONVERSION", "-Wint-conversion") + .put("CLANG_WARN_UNREACHABLE_CODE", "-Wunreachable-code") + .put("GCC_WARN_ABOUT_RETURN_TYPE", "-Wmismatched-return-types") + .put("GCC_WARN_UNDECLARED_SELECTOR", "-Wundeclared-selector") + .put("GCC_WARN_UNINITIALIZED_AUTOS", "-Wuninitialized") + .put("GCC_WARN_UNUSED_FUNCTION", "-Wunused-function") + .put("GCC_WARN_UNUSED_VARIABLE", "-Wunused-variable") + .build(); + + static final ImmutableList<String> DEFAULT_LINKER_FLAGS = ImmutableList.of("-ObjC"); + + private IosSdkCommands() { + throw new UnsupportedOperationException("static-only"); + } + + private static String platformDir(ObjcConfiguration configuration) { + return DEVELOPER_DIR + "/Platforms/" + configuration.getPlatform().getNameInPlist() + + ".platform"; + } + + public static String sdkDir(ObjcConfiguration configuration) { + return platformDir(configuration) + "/Developer/SDKs/" + + configuration.getPlatform().getNameInPlist() + configuration.getIosSdkVersion() + ".sdk"; + } + + public static String frameworkDir(ObjcConfiguration configuration) { + return platformDir(configuration) + "/Developer/Library/Frameworks"; + } + + private static Iterable<PathFragment> uniqueParentDirectories(Iterable<PathFragment> paths) { + ImmutableSet.Builder<PathFragment> parents = new ImmutableSet.Builder<>(); + for (PathFragment path : paths) { + parents.add(path.getParentDirectory()); + } + return parents.build(); + } + + public static List<String> commonLinkAndCompileArgsForClang( + ObjcProvider provider, ObjcConfiguration configuration) { + ImmutableList.Builder<String> builder = new ImmutableList.Builder<>(); + if (configuration.getPlatform() == Platform.SIMULATOR) { + builder.add("-mios-simulator-version-min=" + configuration.getMinimumOs()); + } else { + builder.add("-miphoneos-version-min=" + configuration.getMinimumOs()); + } + + if (configuration.generateDebugSymbols()) { + builder.add("-g"); + } + + return builder + .add("-arch", configuration.getIosCpu()) + .add("-isysroot", sdkDir(configuration)) + // TODO(bazel-team): Pass framework search paths to Xcodegen. + .add("-F", sdkDir(configuration) + "/Developer/Library/Frameworks") + // As of sdk8.1, XCTest is in a base Framework dir + .add("-F", frameworkDir(configuration)) + // Add custom (non-SDK) framework search paths. For each framework foo/bar.framework, + // include "foo" as a search path. + .addAll(Interspersing.beforeEach( + "-F", + PathFragment.safePathStrings(uniqueParentDirectories(provider.get(FRAMEWORK_DIR))))) + .build(); + } + + public static Iterable<String> compileArgsForClang(ObjcConfiguration configuration) { + return Iterables.concat( + DEFAULT_WARNINGS.values(), + platformSpecificCompileArgsForClang(configuration) + ); + } + + private static List<String> platformSpecificCompileArgsForClang(ObjcConfiguration configuration) { + switch (configuration.getPlatform()) { + case DEVICE: + return ImmutableList.of(); + case SIMULATOR: + // These are added by Xcode when building, because the simulator is built on OSX + // frameworks so we aim compile to match the OSX objc runtime. + return ImmutableList.of( + "-fexceptions", + "-fasm-blocks", + "-fobjc-abi-version=2", + "-fobjc-legacy-dispatch"); + default: + throw new AssertionError(); + } + } + + public static Iterable<? extends XcodeprojBuildSetting> defaultWarningsForXcode() { + return Iterables.transform(DEFAULT_WARNINGS.keySet(), + new Function<String, XcodeprojBuildSetting>() { + @Override + public XcodeprojBuildSetting apply(String key) { + return XcodeprojBuildSetting.newBuilder().setName(key).setValue("YES").build(); + } + }); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosTest.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosTest.java new file mode 100644 index 0000000000..9e9ddc4ebb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosTest.java @@ -0,0 +1,163 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.objc.ApplicationSupport.LinkedBinary; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkArgs; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkInputs; + +import java.util.ArrayList; +import java.util.List; + +/** + * Contains information needed to create a {@link RuleConfiguredTarget} and invoke test runners + * for some instantiation of this rule. + */ +// TODO(bazel-team): Extract a TestSupport class that takes on most of the logic in this class. +public abstract class IosTest implements RuleConfiguredTargetFactory { + private static final ImmutableList<SdkFramework> AUTOMATIC_SDK_FRAMEWORKS_FOR_XCTEST = + ImmutableList.of(new SdkFramework("XCTest")); + + public static final String TARGET_DEVICE = "target_device"; + public static final String IS_XCTEST = "xctest"; + public static final String XCTEST_APP = "xctest_app"; + + @VisibleForTesting + public static final String REQUIRES_SOURCE_ERROR = + "ios_test requires at least one source file in srcs or non_arc_srcs"; + + /** + * Creates a target, including registering actions, just as {@link #create(RuleContext)} does. + * The difference between {@link #create(RuleContext)} and this method is that this method does + * only what is needed to support tests on the environment besides generate the Xcodeproj file + * and build the app and test {@code .ipa}s. The {@link #create(RuleContext)} method delegates + * to this method. + */ + protected abstract ConfiguredTarget create(RuleContext ruleContext, ObjcCommon common, + XcodeProvider xcodeProvider, NestedSet<Artifact> filesToBuild) throws InterruptedException; + + @Override + public final ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + ObjcCommon common = common(ruleContext); + OptionsProvider optionsProvider = optionsProvider(ruleContext); + + if (!common.getCompilationArtifacts().get().getArchive().isPresent()) { + ruleContext.ruleError(REQUIRES_SOURCE_ERROR); + } + + XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder(); + NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(common.getStoryboards().getOutputZips()) + .addAll(Xcdatamodel.outputZips(common.getDatamodels())); + + XcodeProductType productType; + ExtraLinkArgs extraLinkArgs; + ExtraLinkInputs extraLinkInputs; + if (!isXcTest(ruleContext)) { + productType = XcodeProductType.APPLICATION; + extraLinkArgs = new ExtraLinkArgs(); + extraLinkInputs = new ExtraLinkInputs(); + } else { + productType = XcodeProductType.UNIT_TEST; + XcodeProvider appIpaXcodeProvider = + ruleContext.getPrerequisite(XCTEST_APP, Mode.TARGET, XcodeProvider.class); + xcodeProviderBuilder + .setTestHost(appIpaXcodeProvider) + .setProductType(productType); + + Artifact bundleLoader = xcTestAppProvider(ruleContext).getBundleLoader(); + + // -bundle causes this binary to be linked as a bundle and not require an entry point + // (i.e. main()) + // -bundle_loader causes the code in this test to have access to the symbols in the test rig, + // or more specifically, the flag causes ld to consider the given binary when checking for + // missing symbols. + extraLinkArgs = new ExtraLinkArgs( + "-bundle", + "-bundle_loader", bundleLoader.getExecPathString()); + extraLinkInputs = new ExtraLinkInputs(bundleLoader); + } + + new CompilationSupport(ruleContext) + .registerLinkActions( + common.getObjcProvider(), extraLinkArgs, extraLinkInputs) + .registerJ2ObjcCompileAndArchiveActions(optionsProvider, common.getObjcProvider()) + .registerCompileAndArchiveActions(common, optionsProvider) + .addXcodeSettings(xcodeProviderBuilder, common, optionsProvider) + .validateAttributes(); + + new ApplicationSupport( + ruleContext, common.getObjcProvider(), optionsProvider, LinkedBinary.LOCAL_AND_DEPENDENCIES) + .registerActions() + .addXcodeSettings(xcodeProviderBuilder) + .addFilesToBuild(filesToBuild) + .validateAttributes(); + + new ResourceSupport(ruleContext) + .registerActions(common.getStoryboards()) + .validateAttributes() + .addXcodeSettings(xcodeProviderBuilder); + + new XcodeSupport(ruleContext) + .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), productType) + .addDependencies(xcodeProviderBuilder) + .addFilesToBuild(filesToBuild) + .registerActions(xcodeProviderBuilder.build()); + + return create(ruleContext, common, xcodeProviderBuilder.build(), filesToBuild.build()); + } + + protected static boolean isXcTest(RuleContext ruleContext) { + return ruleContext.attributes().get(IS_XCTEST, Type.BOOLEAN); + } + + private OptionsProvider optionsProvider(RuleContext ruleContext) { + return new OptionsProvider.Builder() + .addCopts(ruleContext.getTokenizedStringListAttr("copts")) + .addInfoplists(ruleContext.getPrerequisiteArtifacts("infoplist", Mode.TARGET).list()) + .addTransitive(Optional.fromNullable( + ruleContext.getPrerequisite("options", Mode.TARGET, OptionsProvider.class))) + .build(); + } + + /** Returns the {@link XcTestAppProvider} of the {@code xctest_app} attribute. */ + private static XcTestAppProvider xcTestAppProvider(RuleContext ruleContext) { + return ruleContext.getPrerequisite(XCTEST_APP, Mode.TARGET, XcTestAppProvider.class); + } + + private ObjcCommon common(RuleContext ruleContext) { + ImmutableList<SdkFramework> extraSdkFrameworks = isXcTest(ruleContext) + ? AUTOMATIC_SDK_FRAMEWORKS_FOR_XCTEST : ImmutableList.<SdkFramework>of(); + List<ObjcProvider> extraDepObjcProviders = new ArrayList<>(); + if (isXcTest(ruleContext)) { + extraDepObjcProviders.add(xcTestAppProvider(ruleContext).getObjcProvider()); + } + + return ObjcLibrary.common(ruleContext, extraSdkFrameworks, /*alwayslink=*/false, + new ObjcLibrary.ExtraImportLibraries(), new ObjcLibrary.Defines(), extraDepObjcProviders); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IterableWrapper.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IterableWrapper.java new file mode 100644 index 0000000000..4bd7b0c06b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IterableWrapper.java @@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.Iterator; + +/** + * Base class for tiny container types that encapsulate an iterable. + */ +abstract class IterableWrapper<E> implements Iterable<E> { + private final Iterable<E> contents; + + IterableWrapper(Iterable<E> contents) { + this.contents = Preconditions.checkNotNull(contents); + } + + IterableWrapper(E... contents) { + this.contents = ImmutableList.copyOf(contents); + } + + @Override + public Iterator<E> iterator() { + return contents.iterator(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcHeaderMappingFileProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcHeaderMappingFileProvider.java new file mode 100644 index 0000000000..93ff980333 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcHeaderMappingFileProvider.java @@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * This provider is exported by java_library rules to supply ObjC header to Java type mapping files + * for J2ObjC translation. J2ObjC needs the mapping files to be able to output translated files with + * correct header import paths in the same directories of the Java source files. + */ +@Immutable +public final class J2ObjcHeaderMappingFileProvider implements TransitiveInfoProvider { + private final NestedSet<Artifact> mappingFiles; + + public J2ObjcHeaderMappingFileProvider(NestedSet<Artifact> mappingFiles) { + this.mappingFiles = mappingFiles; + } + + public NestedSet<Artifact> getMappingFiles() { + return mappingFiles; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSource.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSource.java new file mode 100644 index 0000000000..81ba292314 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSource.java @@ -0,0 +1,124 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Objects; +import com.google.common.collect.Iterators; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * An object that captures information of ObjC files generated by J2ObjC in a single target. + */ +public class J2ObjcSource { + + /** + * Indicates the type of files from which the ObjC files included in {@link J2ObjcSource} are + * generated. + */ + public enum SourceType { + /** + * Indicates the original file type is java source file. + */ + JAVA, + + /** + * Indicates the original file type is proto file. + */ + PROTO; + } + + private final Label targetLabel; + private final Iterable<Artifact> objcSrcs; + private final Iterable<Artifact> objcHdrs; + private final PathFragment objcFilePath; + private final SourceType sourceType; + + /** + * Constructs a J2ObjcSource containing target information for j2objc transpilation. + * + * @param targetLabel the @{code Label} of the associated target. + * @param objcSrcs the {@code Iterable} containing objc source files generated by J2ObjC + * @param objcHdrs the {@code Iterable} containing objc header files generated by J2ObjC + * @param objcFilePath the {@code PathFragment} under which all the generated objc files are. It + * can be used as header search path for objc compilations. + * @param sourceType the type of files from which the ObjC files are generated. + */ + public J2ObjcSource(Label targetLabel, Iterable<Artifact> objcSrcs, + Iterable<Artifact> objcHdrs, PathFragment objcFilePath, SourceType sourceType) { + this.targetLabel = targetLabel; + this.objcSrcs = objcSrcs; + this.objcHdrs = objcHdrs; + this.objcFilePath = objcFilePath; + this.sourceType = sourceType; + } + + /** + * Returns the label of the associated target. + */ + public Label getTargetLabel() { + return targetLabel; + } + + /** + * Returns the objc source files generated by J2ObjC. + */ + public Iterable<Artifact> getObjcSrcs() { + return objcSrcs; + } + + /* + * Returns the objc header files generated by J2ObjC + */ + public Iterable<Artifact> getObjcHdrs() { + return objcHdrs; + } + + /** + * Returns the {@code PathFragment} which represents a directory where the generated ObjC files + * reside and which can also be used as header search path in ObjC compilation. + */ + public PathFragment getObjcFilePath() { + return objcFilePath; + } + + /** + * Returns the type of files from which the ObjC files inside this object are generated. + */ + public SourceType getSourceType() { + return sourceType; + } + + @Override + public final boolean equals(Object other) { + if (!(other instanceof J2ObjcSource)) { + return false; + } + + J2ObjcSource that = (J2ObjcSource) other; + return Objects.equal(this.targetLabel, that.targetLabel) + && Iterators.elementsEqual(this.objcSrcs.iterator(), that.objcSrcs.iterator()) + && Iterators.elementsEqual(this.objcHdrs.iterator(), that.objcHdrs.iterator()) + && Objects.equal(this.objcFilePath, that.objcFilePath) + && this.sourceType == that.sourceType; + } + + @Override + public int hashCode() { + return Objects.hashCode(targetLabel, objcSrcs, objcHdrs, objcFilePath, sourceType); + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSrcsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSrcsProvider.java new file mode 100644 index 0000000000..1e577c5939 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSrcsProvider.java @@ -0,0 +1,45 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * This provider is exported by java_library rules to supply J2ObjC-translated ObjC sources to + * objc_binary for compilation and linking. + */ +@Immutable +public final class J2ObjcSrcsProvider implements TransitiveInfoProvider { + private final NestedSet<J2ObjcSource> srcs; + private final boolean hasProtos; + + public J2ObjcSrcsProvider(NestedSet<J2ObjcSource> srcs, boolean hasProtos) { + this.srcs = srcs; + this.hasProtos = hasProtos; + } + + public NestedSet<J2ObjcSource> getSrcs() { + return srcs; + } + + /** + * Returns whether the translated source files in the provider has proto files. + */ + public boolean hasProtos() { + return hasProtos; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcActionsBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcActionsBuilder.java new file mode 100644 index 0000000000..26742d9200 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcActionsBuilder.java @@ -0,0 +1,599 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.IosSdkCommands.BIN_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.ASSET_CATALOG; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.DEFINE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FORCE_LOAD_LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_FILE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.Flag.USES_CPP; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.HEADER; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.INCLUDE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_DYLIB; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_FRAMEWORK; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.WEAK_SDK_FRAMEWORK; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCASSETS_DIR; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.io.ByteSource; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionRegistry; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext; +import com.google.devtools.build.lib.analysis.actions.BinaryFileWriteAction; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.util.LazyString; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.common.TargetDeviceFamily; +import com.google.devtools.build.xcode.util.Interspersing; +import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +/** + * Object that creates actions used by Objective-C rules. + */ +final class ObjcActionsBuilder { + private final ActionConstructionContext context; + private final IntermediateArtifacts intermediateArtifacts; + private final ObjcConfiguration objcConfiguration; + private final BuildConfiguration buildConfiguration; + private final ActionRegistry actionRegistry; + + ObjcActionsBuilder(ActionConstructionContext context, IntermediateArtifacts intermediateArtifacts, + ObjcConfiguration objcConfiguration, BuildConfiguration buildConfiguration, + ActionRegistry actionRegistry) { + this.context = Preconditions.checkNotNull(context); + this.intermediateArtifacts = Preconditions.checkNotNull(intermediateArtifacts); + this.objcConfiguration = Preconditions.checkNotNull(objcConfiguration); + this.buildConfiguration = Preconditions.checkNotNull(buildConfiguration); + this.actionRegistry = Preconditions.checkNotNull(actionRegistry); + } + + /** + * Creates a new spawn action builder that requires a darwin architecture to run. + */ + // TODO(bazel-team): Use everywhere we currently set the execution info manually. + static SpawnAction.Builder spawnOnDarwinActionBuilder() { + return new SpawnAction.Builder() + .setExecutionInfo(ImmutableMap.of(ExecutionRequirements.REQUIRES_DARWIN, "")); + } + + static final PathFragment JAVA = new PathFragment("/usr/bin/java"); + static final PathFragment CLANG = new PathFragment(BIN_DIR + "/clang"); + static final PathFragment CLANG_PLUSPLUS = new PathFragment(BIN_DIR + "/clang++"); + static final PathFragment LIBTOOL = new PathFragment(BIN_DIR + "/libtool"); + static final PathFragment IBTOOL = new PathFragment(IosSdkCommands.IBTOOL_PATH); + static final PathFragment DSYMUTIL = new PathFragment(BIN_DIR + "/dsymutil"); + static final PathFragment LIPO = new PathFragment(BIN_DIR + "/lipo"); + + // TODO(bazel-team): Reference a rule target rather than a jar file when Darwin runfiles work + // better. + private static SpawnAction.Builder spawnJavaOnDarwinActionBuilder(Artifact deployJarArtifact) { + return spawnOnDarwinActionBuilder() + .setExecutable(JAVA) + .addExecutableArguments("-jar", deployJarArtifact.getExecPathString()) + .addInput(deployJarArtifact); + } + + private void registerCompileAction( + Artifact sourceFile, + Artifact objFile, + Optional<Artifact> pchFile, + ObjcProvider objcProvider, + Iterable<String> otherFlags, + OptionsProvider optionsProvider) { + CustomCommandLine.Builder commandLine = new CustomCommandLine.Builder(); + if (ObjcRuleClasses.CPP_SOURCES.matches(sourceFile.getExecPath())) { + commandLine.add("-stdlib=libc++"); + } + commandLine + .add(IosSdkCommands.compileArgsForClang(objcConfiguration)) + .add(IosSdkCommands.commonLinkAndCompileArgsForClang( + objcProvider, objcConfiguration)) + .add(objcConfiguration.getCoptsForCompilationMode()) + .addBeforeEachPath("-iquote", ObjcCommon.userHeaderSearchPaths(buildConfiguration)) + .addBeforeEachExecPath("-include", pchFile.asSet()) + .addBeforeEachPath("-I", objcProvider.get(INCLUDE)) + .add(otherFlags) + .addFormatEach("-D%s", objcProvider.get(DEFINE)) + .add(objcConfiguration.getCopts()) + .add(optionsProvider.getCopts()) + .addExecPath("-c", sourceFile) + .addExecPath("-o", objFile); + + register(spawnOnDarwinActionBuilder() + .setMnemonic("ObjcCompile") + .setExecutable(CLANG) + .setCommandLine(commandLine.build()) + .addInput(sourceFile) + .addOutput(objFile) + .addTransitiveInputs(objcProvider.get(HEADER)) + .addTransitiveInputs(objcProvider.get(FRAMEWORK_FILE)) + .addInputs(pchFile.asSet()) + .build(context)); + } + + private static final ImmutableList<String> ARC_ARGS = ImmutableList.of("-fobjc-arc"); + private static final ImmutableList<String> NON_ARC_ARGS = ImmutableList.of("-fno-objc-arc"); + + /** + * Creates actions to compile each source file individually, and link all the compiled object + * files into a single archive library. + */ + void registerCompileAndArchiveActions(CompilationArtifacts compilationArtifacts, + ObjcProvider objcProvider, OptionsProvider optionsProvider) { + ImmutableList.Builder<Artifact> objFiles = new ImmutableList.Builder<>(); + for (Artifact sourceFile : compilationArtifacts.getSrcs()) { + Artifact objFile = intermediateArtifacts.objFile(sourceFile); + objFiles.add(objFile); + registerCompileAction(sourceFile, objFile, compilationArtifacts.getPchFile(), + objcProvider, ARC_ARGS, optionsProvider); + } + for (Artifact nonArcSourceFile : compilationArtifacts.getNonArcSrcs()) { + Artifact objFile = intermediateArtifacts.objFile(nonArcSourceFile); + objFiles.add(objFile); + registerCompileAction(nonArcSourceFile, objFile, compilationArtifacts.getPchFile(), + objcProvider, NON_ARC_ARGS, optionsProvider); + } + for (Artifact archive : compilationArtifacts.getArchive().asSet()) { + registerAll(archiveActions(context, objFiles.build(), archive, objcConfiguration, + intermediateArtifacts.objList())); + } + } + + private static Iterable<Action> archiveActions( + ActionConstructionContext context, + final Iterable<Artifact> objFiles, + final Artifact archive, + final ObjcConfiguration objcConfiguration, + final Artifact objList) { + + ImmutableList.Builder<Action> actions = new ImmutableList.Builder<>(); + + actions.add(new FileWriteAction( + context.getActionOwner(), objList, joinExecPaths(objFiles), /*makeExecutable=*/ false)); + + actions.add(spawnOnDarwinActionBuilder() + .setMnemonic("ObjcLink") + .setExecutable(LIBTOOL) + .setCommandLine(new CommandLine() { + @Override + public Iterable<String> arguments() { + return new ImmutableList.Builder<String>() + .add("-static") + .add("-filelist").add(objList.getExecPathString()) + .add("-arch_only").add(objcConfiguration.getIosCpu()) + .add("-syslibroot").add(IosSdkCommands.sdkDir(objcConfiguration)) + .add("-o").add(archive.getExecPathString()) + .build(); + } + }) + .addInputs(objFiles) + .addInput(objList) + .addOutput(archive) + .build(context)); + + return actions.build(); + } + + private void register(Action... action) { + actionRegistry.registerAction(action); + } + + private void registerAll(Iterable<? extends Action> actions) { + for (Action action : actions) { + actionRegistry.registerAction(action); + } + } + + private static ByteSource xcodegenControlFileBytes( + final Artifact pbxproj, final XcodeProvider.Project project, final String minimumOs) { + return new ByteSource() { + @Override + public InputStream openStream() { + return XcodeGenProtos.Control.newBuilder() + .setPbxproj(pbxproj.getExecPathString()) + .addAllTarget(project.targets()) + .addBuildSetting(XcodeGenProtos.XcodeprojBuildSetting.newBuilder() + .setName("IPHONEOS_DEPLOYMENT_TARGET") + .setValue(minimumOs) + .build()) + .build() + .toByteString() + .newInput(); + } + }; + } + + /** + * Generates actions needed to create an Xcode project file. + */ + void registerXcodegenActions( + ObjcRuleClasses.Tools baseTools, Artifact pbxproj, XcodeProvider.Project project) { + Artifact controlFile = intermediateArtifacts.pbxprojControlArtifact(); + register(new BinaryFileWriteAction( + context.getActionOwner(), + controlFile, + xcodegenControlFileBytes(pbxproj, project, objcConfiguration.getMinimumOs()), + /*makeExecutable=*/false)); + register(new SpawnAction.Builder() + .setMnemonic("GenerateXcodeproj") + .setExecutable(baseTools.xcodegen()) + .addArgument("--control") + .addInputArgument(controlFile) + .addOutput(pbxproj) + .addTransitiveInputs(project.getInputsToXcodegen()) + .build(context)); + } + + /** + * Creates actions to convert all files specified by the strings attribute into binary format. + */ + private static Iterable<Action> convertStringsActions( + ActionConstructionContext context, + ObjcRuleClasses.Tools baseTools, + StringsFiles stringsFiles) { + ImmutableList.Builder<Action> result = new ImmutableList.Builder<>(); + for (CompiledResourceFile stringsFile : stringsFiles) { + final Artifact original = stringsFile.getOriginal(); + final Artifact bundled = stringsFile.getBundled().getBundled(); + result.add(new SpawnAction.Builder() + .setMnemonic("ConvertStringsPlist") + .setExecutable(baseTools.plmerge()) + .setCommandLine(new CommandLine() { + @Override + public Iterable<String> arguments() { + return ImmutableList.of("--source_file", original.getExecPathString(), + "--out_file", bundled.getExecPathString()); + } + }) + .addInput(original) + .addOutput(bundled) + .build(context)); + } + return result.build(); + } + + private Action[] ibtoolzipAction(ObjcRuleClasses.Tools baseTools, String mnemonic, Artifact input, + Artifact zipOutput, String archiveRoot) { + return spawnJavaOnDarwinActionBuilder(baseTools.actooloribtoolzipDeployJar()) + .setMnemonic(mnemonic) + .setCommandLine(new CustomCommandLine.Builder() + // The next three arguments are positional, i.e. they don't have flags before them. + .addPath(zipOutput.getExecPath()) + .add(archiveRoot) + .addPath(IBTOOL) + + .add("--minimum-deployment-target").add(objcConfiguration.getMinimumOs()) + .addPath(input.getExecPath()) + .build()) + .addOutput(zipOutput) + .addInput(input) + .build(context); + } + + /** + * Creates actions to convert all files specified by the xibs attribute into nib format. + */ + private Iterable<Action> convertXibsActions(ObjcRuleClasses.Tools baseTools, XibFiles xibFiles) { + ImmutableList.Builder<Action> result = new ImmutableList.Builder<>(); + for (Artifact original : xibFiles) { + Artifact zipOutput = intermediateArtifacts.compiledXibFileZip(original); + String archiveRoot = BundleableFile.bundlePath( + FileSystemUtils.replaceExtension(original.getExecPath(), ".nib")); + result.add(ibtoolzipAction(baseTools, "XibCompile", original, zipOutput, archiveRoot)); + } + return result.build(); + } + + /** + * Outputs of an {@code actool} action besides the zip file. + */ + static final class ExtraActoolOutputs extends IterableWrapper<Artifact> { + ExtraActoolOutputs(Artifact... extraActoolOutputs) { + super(extraActoolOutputs); + } + } + + static final class ExtraActoolArgs extends IterableWrapper<String> { + ExtraActoolArgs(Iterable<String> args) { + super(args); + } + + ExtraActoolArgs(String... args) { + super(args); + } + } + + void registerActoolzipAction( + ObjcRuleClasses.Tools tools, + ObjcProvider provider, + Artifact zipOutput, + ExtraActoolOutputs extraActoolOutputs, + ExtraActoolArgs extraActoolArgs, + Set<TargetDeviceFamily> families) { + // TODO(bazel-team): Do not use the deploy jar explicitly here. There is currently a bug where + // we cannot .setExecutable({java_binary target}) and set REQUIRES_DARWIN in the execution info. + // Note that below we set the archive root to the empty string. This means that the generated + // zip file will be rooted at the bundle root, and we have to prepend the bundle root to each + // entry when merging it with the final .ipa file. + register(spawnJavaOnDarwinActionBuilder(tools.actooloribtoolzipDeployJar()) + .setMnemonic("AssetCatalogCompile") + .addTransitiveInputs(provider.get(ASSET_CATALOG)) + .addOutput(zipOutput) + .addOutputs(extraActoolOutputs) + .setCommandLine(actoolzipCommandLine( + objcConfiguration, + provider, + zipOutput, + extraActoolArgs, + ImmutableSet.copyOf(families))) + .build(context)); + } + + private static CommandLine actoolzipCommandLine( + final ObjcConfiguration objcConfiguration, + final ObjcProvider provider, + final Artifact zipOutput, + final ExtraActoolArgs extraActoolArgs, + final ImmutableSet<TargetDeviceFamily> families) { + return new CommandLine() { + @Override + public Iterable<String> arguments() { + ImmutableList.Builder<String> args = new ImmutableList.Builder<String>() + // The next three arguments are positional, i.e. they don't have flags before them. + .add(zipOutput.getExecPathString()) + .add("") // archive root + .add(IosSdkCommands.ACTOOL_PATH) + .add("--platform") + .add(objcConfiguration.getPlatform().getLowerCaseNameInPlist()) + .add("--minimum-deployment-target").add(objcConfiguration.getMinimumOs()); + for (TargetDeviceFamily targetDeviceFamily : families) { + args.add("--target-device").add(targetDeviceFamily.name().toLowerCase(Locale.US)); + } + return args + .addAll(PathFragment.safePathStrings(provider.get(XCASSETS_DIR))) + .addAll(extraActoolArgs) + .build(); + } + }; + } + + void registerIbtoolzipAction(ObjcRuleClasses.Tools tools, Artifact input, Artifact outputZip) { + String archiveRoot = BundleableFile.bundlePath(input.getExecPath()) + "c"; + register(ibtoolzipAction(tools, "StoryboardCompile", input, outputZip, archiveRoot)); + } + + @VisibleForTesting + static Iterable<String> commonMomczipArguments(ObjcConfiguration configuration) { + return ImmutableList.of( + "-XD_MOMC_SDKROOT=" + IosSdkCommands.sdkDir(configuration), + "-XD_MOMC_IOS_TARGET_VERSION=" + configuration.getMinimumOs(), + "-MOMC_PLATFORMS", configuration.getPlatform().getLowerCaseNameInPlist(), + "-XD_MOMC_TARGET_VERSION=10.6"); + } + + private static Iterable<Action> momczipActions(ActionConstructionContext context, + ObjcRuleClasses.Tools baseTools, final ObjcConfiguration objcConfiguration, + Iterable<Xcdatamodel> datamodels) { + ImmutableList.Builder<Action> result = new ImmutableList.Builder<>(); + for (Xcdatamodel datamodel : datamodels) { + final Artifact outputZip = datamodel.getOutputZip(); + final String archiveRoot = datamodel.archiveRootForMomczip(); + final String container = datamodel.getContainer().getSafePathString(); + result.add(spawnJavaOnDarwinActionBuilder(baseTools.momczipDeployJar()) + .setMnemonic("MomCompile") + .addOutput(outputZip) + .addInputs(datamodel.getInputs()) + .setCommandLine(new CommandLine() { + @Override + public Iterable<String> arguments() { + return new ImmutableList.Builder<String>() + .add(outputZip.getExecPathString()) + .add(archiveRoot) + .add(IosSdkCommands.MOMC_PATH) + .addAll(commonMomczipArguments(objcConfiguration)) + .add(container) + .build(); + } + }) + .build(context)); + } + return result.build(); + } + + private static final String FRAMEWORK_SUFFIX = ".framework"; + + /** + * All framework names to pass to the linker using {@code -framework} flags. For a framework in + * the directory foo/bar.framework, the name is "bar". Each framework is found without using the + * full path by means of the framework search paths. The search paths are added by + * {@link IosSdkCommands#commonLinkAndCompileArgsForClang(ObjcProvider, ObjcConfiguration)}). + * + * <p>It's awful that we can't pass the full path to the framework and avoid framework search + * paths, but this is imposed on us by clang. clang does not support passing the full path to the + * framework, so Bazel cannot do it either. + */ + private static Iterable<String> frameworkNames(ObjcProvider provider) { + List<String> names = new ArrayList<>(); + Iterables.addAll(names, SdkFramework.names(provider.get(SDK_FRAMEWORK))); + for (PathFragment frameworkDir : provider.get(FRAMEWORK_DIR)) { + String segment = frameworkDir.getBaseName(); + Preconditions.checkState(segment.endsWith(FRAMEWORK_SUFFIX), + "expect %s to end with %s, but it does not", segment, FRAMEWORK_SUFFIX); + names.add(segment.substring(0, segment.length() - FRAMEWORK_SUFFIX.length())); + } + return names; + } + + static final class ExtraLinkArgs extends IterableWrapper<String> { + ExtraLinkArgs(Iterable<String> args) { + super(args); + } + + ExtraLinkArgs(String... args) { + super(args); + } + } + + static final class ExtraLinkInputs extends IterableWrapper<Artifact> { + ExtraLinkInputs(Iterable<Artifact> inputs) { + super(inputs); + } + + ExtraLinkInputs(Artifact... inputs) { + super(inputs); + } + } + + private static final class LinkCommandLine extends CommandLine { + private static final Joiner commandJoiner = Joiner.on(' '); + private final ObjcProvider objcProvider; + private final ObjcConfiguration objcConfiguration; + private final Artifact linkedBinary; + private final Optional<Artifact> dsymBundle; + private final ExtraLinkArgs extraLinkArgs; + + LinkCommandLine(ObjcConfiguration objcConfiguration, ExtraLinkArgs extraLinkArgs, + ObjcProvider objcProvider, Artifact linkedBinary, Optional<Artifact> dsymBundle) { + this.objcConfiguration = Preconditions.checkNotNull(objcConfiguration); + this.extraLinkArgs = Preconditions.checkNotNull(extraLinkArgs); + this.objcProvider = Preconditions.checkNotNull(objcProvider); + this.linkedBinary = Preconditions.checkNotNull(linkedBinary); + this.dsymBundle = Preconditions.checkNotNull(dsymBundle); + } + + Iterable<String> dylibPaths() { + ImmutableList.Builder<String> args = new ImmutableList.Builder<>(); + for (String dylib : objcProvider.get(SDK_DYLIB)) { + args.add(String.format( + "%s/usr/lib/%s.dylib", IosSdkCommands.sdkDir(objcConfiguration), dylib)); + } + return args.build(); + } + + @Override + public Iterable<String> arguments() { + StringBuilder argumentStringBuilder = new StringBuilder(); + + Iterable<String> archiveExecPaths = Artifact.toExecPaths( + Iterables.concat(objcProvider.get(LIBRARY), objcProvider.get(IMPORTED_LIBRARY))); + commandJoiner.appendTo(argumentStringBuilder, new ImmutableList.Builder<String>() + .add(objcProvider.is(USES_CPP) ? CLANG_PLUSPLUS.toString() : CLANG.toString()) + .addAll(objcProvider.is(USES_CPP) + ? ImmutableList.of("-stdlib=libc++") : ImmutableList.<String>of()) + .addAll(IosSdkCommands.commonLinkAndCompileArgsForClang(objcProvider, objcConfiguration)) + .add("-Xlinker", "-objc_abi_version") + .add("-Xlinker", "2") + .add("-fobjc-link-runtime") + .addAll(IosSdkCommands.DEFAULT_LINKER_FLAGS) + .addAll(Interspersing.beforeEach("-framework", frameworkNames(objcProvider))) + .addAll(Interspersing.beforeEach( + "-weak_framework", SdkFramework.names(objcProvider.get(WEAK_SDK_FRAMEWORK)))) + .add("-o", linkedBinary.getExecPathString()) + .addAll(archiveExecPaths) + .addAll(dylibPaths()) + .addAll(extraLinkArgs) + .build()); + + // Call to dsymutil for debug symbol generation must happen in the link action. + // All debug symbol information is encoded in object files inside archive files. To generate + // the debug symbol bundle, dsymutil will look inside the linked binary for the encoded + // absolute paths to archive files, which are only valid in the link action. + for (Artifact justDsymBundle : dsymBundle.asSet()) { + argumentStringBuilder.append(" "); + commandJoiner.appendTo(argumentStringBuilder, new ImmutableList.Builder<String>() + .add("&&") + .add(DSYMUTIL.toString()) + .add(linkedBinary.getExecPathString()) + .add("-o").add(justDsymBundle.getExecPathString()) + .build()); + } + + return ImmutableList.of(argumentStringBuilder.toString()); + } + } + + /** + * Generates an action to link a binary. + */ + void registerLinkAction(Artifact linkedBinary, ObjcProvider objcProvider, + ExtraLinkArgs extraLinkArgs, ExtraLinkInputs extraLinkInputs, Optional<Artifact> dsymBundle) { + extraLinkArgs = new ExtraLinkArgs(Iterables.concat( + Interspersing.beforeEach( + "-force_load", Artifact.toExecPaths(objcProvider.get(FORCE_LOAD_LIBRARY))), + extraLinkArgs)); + register(spawnOnDarwinActionBuilder() + .setMnemonic("ObjcLink") + .setShellCommand(ImmutableList.of("/bin/bash", "-c")) + .setCommandLine( + new LinkCommandLine(objcConfiguration, extraLinkArgs, objcProvider, linkedBinary, + dsymBundle)) + .addOutput(linkedBinary) + .addOutputs(dsymBundle.asSet()) + .addTransitiveInputs(objcProvider.get(LIBRARY)) + .addTransitiveInputs(objcProvider.get(IMPORTED_LIBRARY)) + .addTransitiveInputs(objcProvider.get(FRAMEWORK_FILE)) + .addInputs(extraLinkInputs) + .build(context)); + } + + static final class StringsFiles extends IterableWrapper<CompiledResourceFile> { + StringsFiles(Iterable<CompiledResourceFile> files) { + super(files); + } + } + + /** + * Registers actions for resource conversion that are needed by all rules that inherit from + * {@link ObjcBase}. + */ + void registerResourceActions(ObjcRuleClasses.Tools baseTools, StringsFiles stringsFiles, + XibFiles xibFiles, Iterable<Xcdatamodel> datamodels) { + registerAll(convertStringsActions(context, baseTools, stringsFiles)); + registerAll(convertXibsActions(baseTools, xibFiles)); + registerAll(momczipActions(context, baseTools, objcConfiguration, datamodels)); + } + + static LazyString joinExecPaths(final Iterable<Artifact> artifacts) { + return new LazyString() { + @Override + public String toString() { + return Artifact.joinExecPaths("\n", artifacts); + } + }; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinary.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinary.java new file mode 100644 index 0000000000..52c897fe23 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinary.java @@ -0,0 +1,147 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LIBRARY; +import static com.google.devtools.build.lib.rules.objc.XcodeProductType.APPLICATION; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.objc.ApplicationSupport.LinkedBinary; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkArgs; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkInputs; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.CompilationAttributes; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.ResourceAttributes; + +/** + * Implementation for the "objc_binary" rule. + */ +public class ObjcBinary implements RuleConfiguredTargetFactory { + + @VisibleForTesting + static final String REQUIRES_AT_LEAST_ONE_LIBRARY_OR_SOURCE_FILE = + "At least one library dependency or source file is required."; + + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + ObjcCommon common = common(ruleContext); + OptionsProvider optionsProvider = optionsProvider(ruleContext); + + ObjcProvider objcProvider = common.getObjcProvider(); + if (!hasLibraryOrSources(objcProvider)) { + ruleContext.ruleError(REQUIRES_AT_LEAST_ONE_LIBRARY_OR_SOURCE_FILE); + return null; + } + + XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder(); + NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.stableOrder(); + + new CompilationSupport(ruleContext) + .registerJ2ObjcCompileAndArchiveActions(optionsProvider, common.getObjcProvider()) + .registerCompileAndArchiveActions(common, optionsProvider) + .addXcodeSettings(xcodeProviderBuilder, common, optionsProvider) + .registerLinkActions(common.getObjcProvider(), new ExtraLinkArgs(), new ExtraLinkInputs()) + .validateAttributes(); + + // TODO(bazel-team): Remove once all bundle users are migrated to ios_application. + ApplicationSupport applicationSupport = new ApplicationSupport( + ruleContext, common.getObjcProvider(), optionsProvider, LinkedBinary.LOCAL_AND_DEPENDENCIES) + .registerActions() + .addXcodeSettings(xcodeProviderBuilder) + .addFilesToBuild(filesToBuild) + .validateAttributes(); + + new ResourceSupport(ruleContext) + .registerActions(common.getStoryboards()) + .validateAttributes() + .addXcodeSettings(xcodeProviderBuilder); + + XcodeSupport xcodeSupport = new XcodeSupport(ruleContext) + // TODO(bazel-team): Use LIBRARY_STATIC as parameter instead of APPLICATION once objc_binary + // no longer creates an application bundle + .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), APPLICATION) + .addDependencies(xcodeProviderBuilder) + .addFilesToBuild(filesToBuild); + XcodeProvider xcodeProvider = xcodeProviderBuilder.build(); + xcodeSupport.registerActions(xcodeProvider); + + // TODO(bazel-team): Stop exporting an XcTestAppProvider once objc_binary no longer creates an + // application bundle. + return common.configuredTarget( + filesToBuild.build(), + Optional.of(xcodeProvider), + Optional.<ObjcProvider>absent(), + Optional.of(applicationSupport.xcTestAppProvider()), + Optional.<J2ObjcSrcsProvider>absent()); + } + + private OptionsProvider optionsProvider(RuleContext ruleContext) { + return new OptionsProvider.Builder() + .addCopts(ruleContext.getTokenizedStringListAttr("copts")) + .addInfoplists(ruleContext.getPrerequisiteArtifacts("infoplist", Mode.TARGET).list()) + .addTransitive(Optional.fromNullable( + ruleContext.getPrerequisite("options", Mode.TARGET, OptionsProvider.class))) + .build(); + } + + private boolean hasLibraryOrSources(ObjcProvider objcProvider) { + return !Iterables.isEmpty(objcProvider.get(LIBRARY)) // Includes sources from this target. + || !Iterables.isEmpty(objcProvider.get(IMPORTED_LIBRARY)); + } + + private ObjcCommon common(RuleContext ruleContext) { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + CompilationArtifacts compilationArtifacts = + CompilationSupport.compilationArtifacts(ruleContext); + + return new ObjcCommon.Builder(ruleContext) + .setCompilationAttributes(new CompilationAttributes(ruleContext)) + .setResourceAttributes(new ResourceAttributes(ruleContext)) + .setCompilationArtifacts(compilationArtifacts) + .addDepObjcProviders(ruleContext.getPrerequisites("deps", Mode.TARGET, ObjcProvider.class)) + .addDepObjcProviders( + ruleContext.getPrerequisites("bundles", Mode.TARGET, ObjcProvider.class)) + .addNonPropagatedDepObjcProviders( + ruleContext.getPrerequisites("non_propagated_deps", Mode.TARGET, ObjcProvider.class)) + .setIntermediateArtifacts(intermediateArtifacts) + .setAlwayslink(false) + .addExtraImportLibraries(j2ObjcLibraries(ruleContext)) + .setLinkedBinary(intermediateArtifacts.singleArchitectureBinary()) + .build(); + } + + private Iterable<Artifact> j2ObjcLibraries(RuleContext ruleContext) { + J2ObjcSrcsProvider j2ObjcSrcsProvider = ObjcRuleClasses.j2ObjcSrcsProvider(ruleContext); + ImmutableList.Builder<Artifact> j2objcLibraries = ImmutableList.builder(); + + // TODO(bazel-team): Refactor the code to stop flattening the nested set here. + for (J2ObjcSource j2ObjcSource : j2ObjcSrcsProvider.getSrcs()) { + j2objcLibraries.add( + ObjcRuleClasses.j2objcIntermediateArtifacts(ruleContext, j2ObjcSource).archive()); + } + + return j2objcLibraries.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinaryRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinaryRule.java new file mode 100644 index 0000000000..c9fc57e36c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinaryRule.java @@ -0,0 +1,62 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for objc_binary. + */ +@BlazeRule(name = "objc_binary", + factoryClass = ObjcBinary.class, + ancestors = { ObjcLibraryRule.class, IosApplicationRule.class }) +public class ObjcBinaryRule implements RuleDefinition { + + @Override + public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) { + return builder + // TODO(bazel-team): Remove bundling functionality (dependency on IosApplicationRule). + /*<!-- #BLAZE_RULE(objc_binary).IMPLICIT_OUTPUTS --> + <ul> + <li><code><var>name</var>.ipa</code>: the application bundle as an <code>.ipa</code> + file</li> + <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: An Xcode project file which + can be used to develop or build on a Mac.</li> + </ul> + <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/ + .setImplicitOutputsFunction( + ImplicitOutputsFunction.fromFunctions(ApplicationSupport.IPA, XcodeSupport.PBXPROJ)) + .removeAttribute("binary") + .removeAttribute("alwayslink") + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_binary, TYPE = BINARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule produces an application bundle by linking one or more Objective-C libraries.</p> + +${IMPLICIT_OUTPUTS} + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/ diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundle.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundle.java new file mode 100644 index 0000000000..6585cbaf73 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundle.java @@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.collect.nestedset.Order.STABLE_ORDER; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * Implementation for {@code objc_bundle}. + */ +public class ObjcBundle implements RuleConfiguredTargetFactory { + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + ObjcCommon common = new ObjcCommon.Builder(ruleContext).build(); + + ImmutableList<Artifact> bundleImports = ruleContext + .getPrerequisiteArtifacts("bundle_imports", Mode.TARGET).list(); + Iterable<String> bundleImportErrors = + ObjcCommon.notInContainerErrors(bundleImports, ObjcCommon.BUNDLE_CONTAINER_TYPE); + for (String error : bundleImportErrors) { + ruleContext.attributeError("bundle_imports", error); + } + + return common.configuredTarget( + /*filesToBuild=*/NestedSetBuilder.<Artifact>emptySet(STABLE_ORDER), + Optional.<XcodeProvider>absent(), + Optional.of(common.getObjcProvider()), + Optional.<XcTestAppProvider>absent(), + Optional.<J2ObjcSrcsProvider>absent()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibrary.java new file mode 100644 index 0000000000..0e7f6b0962 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibrary.java @@ -0,0 +1,103 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.NESTED_BUNDLE; +import static com.google.devtools.build.lib.rules.objc.XcodeProductType.BUNDLE; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.ResourceAttributes; +import com.google.devtools.build.xcode.common.TargetDeviceFamily; + +/** + * Implementation for {@code objc_bundle_library}. + */ +public class ObjcBundleLibrary implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + ObjcCommon common = common(ruleContext); + OptionsProvider optionsProvider = optionsProvider(ruleContext); + + Bundling bundling = bundling(ruleContext, common, optionsProvider); + + XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder(); + NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.stableOrder(); + + // TODO(bazel-team): Figure out if the target device is important, and what to set it to. It may + // have to inherit this from the binary being built. As of this writing, this is only used for + // asset catalogs compilation (actool). + new BundleSupport(ruleContext, ImmutableSet.of(TargetDeviceFamily.IPHONE), bundling) + .registerActions(common.getObjcProvider()) + .addXcodeSettings(xcodeProviderBuilder); + + new ResourceSupport(ruleContext) + .validateAttributes() + .registerActions(common.getStoryboards()) + .addXcodeSettings(xcodeProviderBuilder); + + new XcodeSupport(ruleContext) + .addFilesToBuild(filesToBuild) + .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), BUNDLE) + .registerActions(xcodeProviderBuilder.build()); + + ObjcProvider nestedBundleProvider = new ObjcProvider.Builder() + .add(NESTED_BUNDLE, bundling) + .build(); + + return common.configuredTarget( + filesToBuild.build(), + Optional.of(xcodeProviderBuilder.build()), + Optional.of(nestedBundleProvider), + Optional.<XcTestAppProvider>absent(), + Optional.<J2ObjcSrcsProvider>absent()); + } + + private OptionsProvider optionsProvider(RuleContext ruleContext) { + return new OptionsProvider.Builder() + .addInfoplists(ruleContext.getPrerequisiteArtifacts("infoplist", Mode.TARGET).list()) + .build(); + } + + private Bundling bundling( + RuleContext ruleContext, ObjcCommon common, OptionsProvider optionsProvider) { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + return new Bundling.Builder() + .setName(ruleContext.getLabel().getName()) + .setBundleDirSuffix(".bundle") + .setObjcProvider(common.getObjcProvider()) + .setInfoplistMerging( + BundleSupport.infoPlistMerging(ruleContext, common.getObjcProvider(), optionsProvider)) + .setIntermediateArtifacts(intermediateArtifacts) + .build(); + } + + private ObjcCommon common(RuleContext ruleContext) { + return new ObjcCommon.Builder(ruleContext) + .setResourceAttributes(new ResourceAttributes(ruleContext)) + .addDepObjcProviders( + ruleContext.getPrerequisites("bundles", Mode.TARGET, ObjcProvider.class)) + .setIntermediateArtifacts(ObjcRuleClasses.intermediateArtifacts(ruleContext)) + .build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibraryRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibraryRule.java new file mode 100644 index 0000000000..b9405a651f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibraryRule.java @@ -0,0 +1,67 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for objc_bundle_library. + */ +@BlazeRule(name = "objc_bundle_library", + factoryClass = ObjcBundleLibrary.class, + ancestors = { ObjcRuleClasses.ObjcBaseResourcesRule.class, + ObjcRuleClasses.ObjcHasInfoplistRule.class }) +public class ObjcBundleLibraryRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /*<!-- #BLAZE_RULE(objc_bundle_library).IMPLICIT_OUTPUTS --> + <ul> + <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: An Xcode project file which + can be used to develop or build on a Mac.</li> + </ul> + <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/ + .setImplicitOutputsFunction(ImplicitOutputsFunction.fromFunctions(XcodeSupport.PBXPROJ)) + /* <!-- #BLAZE_RULE(objc_bundle_library).ATTRIBUTE(bundles) --> + The list of bundle targets that this target requires to be included in the final bundle. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("bundles", LABEL_LIST) + .direct_compile_time_input() + .allowedRuleClasses("objc_bundle", "objc_bundle_library") + .allowedFileTypes()) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_bundle_library, TYPE = LIBRARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule encapsulates a library which is provided to dependers as a bundle. +A <code>objc_bundle_library</code>'s resources are put in a nested bundle in +the final iOS application. + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/ diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleRule.java new file mode 100644 index 0000000000..9be0d04831 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleRule.java @@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.util.FileTypeSet; + +/** + * Rule definition for objc_bundle. + */ +@BlazeRule(name = "objc_bundle", + factoryClass = ObjcBundle.class, + ancestors = { BaseRuleClasses.BaseRule.class }) +public class ObjcBundleRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE(objc_bundle).ATTRIBUTE(bundle_imports) --> + The list of files under a <code>.bundle</code> directory which are + provided to Objective-C targets that depend on this target. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("bundle_imports", LABEL_LIST) + .allowedFileTypes(FileTypeSet.ANY_FILE) + .nonEmpty()) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_bundle, TYPE = LIBRARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule encapsulates an already-built bundle. It is defined by a list of +files in one or more <code>.bundle</code> directories. + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/ diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommandLineOptions.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommandLineOptions.java new file mode 100644 index 0000000000..f85609acfe --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommandLineOptions.java @@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.xcode.common.BuildOptionsUtil.DEFAULT_OPTIONS_NAME; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.analysis.config.FragmentOptions; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.common.options.Option; + +import java.util.List; + +/** + * Command-line options for building Objective-C targets. + */ +public class + ObjcCommandLineOptions extends FragmentOptions { + @Option(name = "ios_sdk_version", + defaultValue = DEFAULT_SDK_VERSION, + category = "undocumented", + help = "Specifies the version of the iOS SDK to use to build iOS applications." + ) + public String iosSdkVersion; + + @VisibleForTesting static final String DEFAULT_SDK_VERSION = "8.1"; + + @Option(name = "ios_simulator_version", + defaultValue = "7.1", + category = "undocumented", + help = "The version of iOS to run on the simulator when running tests. This is ignored if the" + + " ios_test rule specifies the target device.", + deprecationWarning = "This flag is deprecated in favor of the target_device attribute and" + + " will eventually removed.") + public String iosSimulatorVersion; + + @Option(name = "ios_cpu", + defaultValue = "i386", + category = "undocumented", + help = "Specifies to target CPU of iOS compilation.") + public String iosCpu; + + @Option(name = "xcode_options", + defaultValue = DEFAULT_OPTIONS_NAME, + category = "undocumented", + help = "Specifies the name of the build settings to use.") + public String xcodeOptions; + + @Option(name = "objc_generate_debug_symbols", + defaultValue = "false", + category = "undocumented", + help = "Specifies whether to generate debug symbol(.dSYM) file.") + public boolean generateDebugSymbols; + + @Option(name = "objccopt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to Objective C compilation.") + public List<String> copts; + + @Option(name = "ios_minimum_os", + defaultValue = DEFAULT_MINIMUM_IOS, + category = "flags", + help = "Minimum compatible iOS version for target simulators and devices.") + public String iosMinimumOs; + + @VisibleForTesting static final String DEFAULT_MINIMUM_IOS = "7.0"; + + @Override + public void addAllLabels(Multimap<String, Label> labelMap) {} +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommon.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommon.java new file mode 100644 index 0000000000..ade86eeb12 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommon.java @@ -0,0 +1,620 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.ASSET_CATALOG; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_FILE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_IMPORT_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.DEFINE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FLAG; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FORCE_LOAD_FOR_XCODEGEN; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FORCE_LOAD_LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_FILE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.Flag.USES_CPP; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.GENERAL_RESOURCE_FILE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.HEADER; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.INCLUDE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LINKED_BINARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.MERGE_ZIP; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_DYLIB; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_FRAMEWORK; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.WEAK_SDK_FRAMEWORK; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCASSETS_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCDATAMODEL; +import static com.google.devtools.build.lib.vfs.PathFragment.TO_PATH_FRAGMENT; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CcCommon; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.util.Interspersing; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Contains information common to multiple objc_* rules, and provides a unified API for extracting + * and accessing it. + */ +// TODO(bazel-team): Decompose and subsume area-specific logic and data into the various *Support +// classes. Make sure to distinguish rule output (providers, runfiles, ...) from intermediate, +// rule-internal information. Any provider created by a rule should not be read, only published. +public final class ObjcCommon { + /** + * Provides a way to access attributes that are common to all compilation rules that inherit from + * {@link ObjcRuleClasses.ObjcCompilationRule}. + */ + // TODO(bazel-team): Delete and move into support-specific attributes classes once ObjcCommon is + // gone. + static final class CompilationAttributes { + private final RuleContext ruleContext; + private final ObjcSdkFrameworks.Attributes sdkFrameworkAttributes; + + CompilationAttributes(RuleContext ruleContext) { + this.ruleContext = Preconditions.checkNotNull(ruleContext); + this.sdkFrameworkAttributes = new ObjcSdkFrameworks.Attributes(ruleContext); + } + + ImmutableList<Artifact> hdrs() { + return ImmutableList.copyOf(CcCommon.getHeaders(ruleContext)); + } + + Iterable<PathFragment> includes() { + return Iterables.transform( + ruleContext.attributes().get("includes", Type.STRING_LIST), + PathFragment.TO_PATH_FRAGMENT); + } + + Iterable<PathFragment> sdkIncludes() { + return Iterables.transform( + ruleContext.attributes().get("sdk_includes", Type.STRING_LIST), + PathFragment.TO_PATH_FRAGMENT); + } + + /** + * Returns the value of the sdk_frameworks attribute plus frameworks that are included + * automatically. + */ + ImmutableSet<SdkFramework> sdkFrameworks() { + return sdkFrameworkAttributes.sdkFrameworks(); + } + + /** + * Returns the value of the weak_sdk_frameworks attribute. + */ + ImmutableSet<SdkFramework> weakSdkFrameworks() { + return sdkFrameworkAttributes.weakSdkFrameworks(); + } + + /** + * Returns the value of the sdk_dylibs attribute. + */ + ImmutableSet<String> sdkDylibs() { + return sdkFrameworkAttributes.sdkDylibs(); + } + + /** + * Returns the exec paths of all header search paths that should be added to this target and + * dependers on this target, obtained from the {@code includes} attribute. + */ + ImmutableList<PathFragment> headerSearchPaths() { + ImmutableList.Builder<PathFragment> paths = new ImmutableList.Builder<>(); + PathFragment packageFragment = ruleContext.getLabel().getPackageFragment(); + List<PathFragment> rootFragments = ImmutableList.of( + packageFragment, + ruleContext.getConfiguration().getGenfilesFragment().getRelative(packageFragment)); + + Iterable<PathFragment> relativeIncludes = + Iterables.filter(includes(), Predicates.not(PathFragment.IS_ABSOLUTE)); + for (PathFragment include : relativeIncludes) { + for (PathFragment rootFragment : rootFragments) { + paths.add(rootFragment.getRelative(include).normalize()); + } + } + return paths.build(); + } + } + + /** + * Provides a way to access attributes that are common to all resources rules that inherit from + * {@link ObjcRuleClasses.ObjcBaseResourcesRule}. + */ + // TODO(bazel-team): Delete and move into support-specific attributes classes once ObjcCommon is + // gone. + static final class ResourceAttributes { + private final RuleContext ruleContext; + + ResourceAttributes(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + ImmutableList<Artifact> strings() { + return ruleContext.getPrerequisiteArtifacts("strings", Mode.TARGET).list(); + } + + ImmutableList<Artifact> xibs() { + return ruleContext.getPrerequisiteArtifacts("xibs", Mode.TARGET) + .errorsForNonMatching(ObjcRuleClasses.XIB_TYPE) + .list(); + } + + ImmutableList<Artifact> storyboards() { + return ruleContext.getPrerequisiteArtifacts("storyboards", Mode.TARGET).list(); + } + + ImmutableList<Artifact> resources() { + return ruleContext.getPrerequisiteArtifacts("resources", Mode.TARGET).list(); + } + + ImmutableList<Artifact> datamodels() { + return ruleContext.getPrerequisiteArtifacts("datamodels", Mode.TARGET).list(); + } + + ImmutableList<Artifact> assetCatalogs() { + return ruleContext.getPrerequisiteArtifacts("asset_catalogs", Mode.TARGET).list(); + } + } + + static class Builder { + private RuleContext context; + private Optional<CompilationAttributes> compilationAttributes = Optional.absent(); + private Optional<ResourceAttributes> resourceAttributes = Optional.absent(); + private Iterable<SdkFramework> extraSdkFrameworks = ImmutableList.of(); + private Iterable<SdkFramework> extraWeakSdkFrameworks = ImmutableList.of(); + private Iterable<String> extraSdkDylibs = ImmutableList.of(); + private Iterable<Artifact> frameworkImports = ImmutableList.of(); + private Optional<CompilationArtifacts> compilationArtifacts = Optional.absent(); + private Iterable<ObjcProvider> depObjcProviders = ImmutableList.of(); + private Iterable<ObjcProvider> directDepObjcProviders = ImmutableList.of(); + private Iterable<String> defines = ImmutableList.of(); + private Iterable<PathFragment> userHeaderSearchPaths = ImmutableList.of(); + private Iterable<Artifact> headers = ImmutableList.of(); + private IntermediateArtifacts intermediateArtifacts; + private boolean alwayslink; + private Iterable<Artifact> extraImportLibraries = ImmutableList.of(); + private Optional<Artifact> linkedBinary = Optional.absent(); + + Builder(RuleContext context) { + this.context = Preconditions.checkNotNull(context); + } + + public Builder setCompilationAttributes(CompilationAttributes baseCompilationAttributes) { + Preconditions.checkState(!this.compilationAttributes.isPresent(), + "compilationAttributes is already set to: %s", this.compilationAttributes); + this.compilationAttributes = Optional.of(baseCompilationAttributes); + return this; + } + + public Builder setResourceAttributes(ResourceAttributes baseResourceAttributes) { + Preconditions.checkState(!this.resourceAttributes.isPresent(), + "resourceAttributes is already set to: %s", this.resourceAttributes); + this.resourceAttributes = Optional.of(baseResourceAttributes); + return this; + } + + Builder addExtraSdkFrameworks(Iterable<SdkFramework> extraSdkFrameworks) { + this.extraSdkFrameworks = Iterables.concat(this.extraSdkFrameworks, extraSdkFrameworks); + return this; + } + + Builder addExtraWeakSdkFrameworks(Iterable<SdkFramework> extraWeakSdkFrameworks) { + this.extraWeakSdkFrameworks = + Iterables.concat(this.extraWeakSdkFrameworks, extraWeakSdkFrameworks); + return this; + } + + Builder addExtraSdkDylibs(Iterable<String> extraSdkDylibs) { + this.extraSdkDylibs = Iterables.concat(this.extraSdkDylibs, extraSdkDylibs); + return this; + } + + Builder addFrameworkImports(Iterable<Artifact> frameworkImports) { + this.frameworkImports = Iterables.concat(this.frameworkImports, frameworkImports); + return this; + } + + Builder setCompilationArtifacts(CompilationArtifacts compilationArtifacts) { + Preconditions.checkState(!this.compilationArtifacts.isPresent(), + "compilationArtifacts is already set to: %s", this.compilationArtifacts); + this.compilationArtifacts = Optional.of(compilationArtifacts); + return this; + } + + /** + * Add providers which will be exposed both to the declaring rule and to any dependers on the + * declaring rule. + */ + Builder addDepObjcProviders(Iterable<ObjcProvider> depObjcProviders) { + this.depObjcProviders = Iterables.concat(this.depObjcProviders, depObjcProviders); + return this; + } + + /** + * Add providers which will only be used by the declaring rule, and won't be propagated to any + * dependers on the declaring rule. + */ + Builder addNonPropagatedDepObjcProviders(Iterable<ObjcProvider> directDepObjcProviders) { + this.directDepObjcProviders = Iterables.concat( + this.directDepObjcProviders, directDepObjcProviders); + return this; + } + + public Builder addUserHeaderSearchPaths(Iterable<PathFragment> userHeaderSearchPaths) { + this.userHeaderSearchPaths = + Iterables.concat(this.userHeaderSearchPaths, userHeaderSearchPaths); + return this; + } + + public Builder addDefines(Iterable<String> defines) { + this.defines = Iterables.concat(this.defines, defines); + return this; + } + + public Builder addHeaders(Iterable<Artifact> headers) { + this.headers = Iterables.concat(this.headers, headers); + return this; + } + + Builder setIntermediateArtifacts(IntermediateArtifacts intermediateArtifacts) { + this.intermediateArtifacts = intermediateArtifacts; + return this; + } + + Builder setAlwayslink(boolean alwayslink) { + this.alwayslink = alwayslink; + return this; + } + + /** + * Adds additional static libraries to be linked into the final ObjC application bundle. + */ + Builder addExtraImportLibraries(Iterable<Artifact> extraImportLibraries) { + this.extraImportLibraries = Iterables.concat(this.extraImportLibraries, extraImportLibraries); + return this; + } + + /** + * Sets a linked binary generated by this rule to be propagated to dependers. + */ + Builder setLinkedBinary(Artifact linkedBinary) { + this.linkedBinary = Optional.of(linkedBinary); + return this; + } + + ObjcCommon build() { + Iterable<BundleableFile> bundleImports = BundleableFile.bundleImportsFromRule(context); + + ObjcProvider.Builder objcProvider = new ObjcProvider.Builder() + .addAll(IMPORTED_LIBRARY, extraImportLibraries) + .addAll(BUNDLE_FILE, bundleImports) + .addAll(BUNDLE_IMPORT_DIR, + uniqueContainers(BundleableFile.toArtifacts(bundleImports), BUNDLE_CONTAINER_TYPE)) + .addAll(SDK_FRAMEWORK, extraSdkFrameworks) + .addAll(WEAK_SDK_FRAMEWORK, extraWeakSdkFrameworks) + .addAll(SDK_DYLIB, extraSdkDylibs) + .addAll(FRAMEWORK_FILE, frameworkImports) + .addAll(FRAMEWORK_DIR, uniqueContainers(frameworkImports, FRAMEWORK_CONTAINER_TYPE)) + .addAll(INCLUDE, userHeaderSearchPaths) + .addAll(DEFINE, defines) + .addAll(HEADER, headers) + .addTransitiveAndPropagate(depObjcProviders) + .addTransitiveWithoutPropagating(directDepObjcProviders); + + Storyboards storyboards; + Iterable<Xcdatamodel> datamodels; + if (compilationAttributes.isPresent()) { + CompilationAttributes attributes = compilationAttributes.get(); + ObjcConfiguration objcConfiguration = ObjcRuleClasses.objcConfiguration(context); + Iterable<PathFragment> sdkIncludes = Iterables.transform( + Interspersing.prependEach( + IosSdkCommands.sdkDir(objcConfiguration) + "/usr/include/", + PathFragment.safePathStrings(attributes.sdkIncludes())), + TO_PATH_FRAGMENT); + objcProvider + .addAll(HEADER, attributes.hdrs()) + .addAll(INCLUDE, attributes.headerSearchPaths()) + .addAll(INCLUDE, sdkIncludes) + .addAll(SDK_FRAMEWORK, attributes.sdkFrameworks()) + .addAll(WEAK_SDK_FRAMEWORK, attributes.weakSdkFrameworks()) + .addAll(SDK_DYLIB, attributes.sdkDylibs()); + } + + if (resourceAttributes.isPresent()) { + ResourceAttributes attributes = resourceAttributes.get(); + storyboards = Storyboards.fromInputs(attributes.storyboards(), intermediateArtifacts); + datamodels = Xcdatamodels.xcdatamodels(intermediateArtifacts, attributes.datamodels()); + Iterable<CompiledResourceFile> compiledResources = + CompiledResourceFile.fromStringsFiles(intermediateArtifacts, attributes.strings()); + XibFiles xibFiles = new XibFiles(attributes.xibs()); + + objcProvider + .addTransitiveAndPropagate(MERGE_ZIP, storyboards.getOutputZips()) + .addAll(MERGE_ZIP, xibFiles.compiledZips(intermediateArtifacts)) + .addAll(GENERAL_RESOURCE_FILE, storyboards.getInputs()) + .addAll(GENERAL_RESOURCE_FILE, attributes.resources()) + .addAll(GENERAL_RESOURCE_FILE, attributes.strings()) + .addAll(GENERAL_RESOURCE_FILE, attributes.xibs()) + .addAll(BUNDLE_FILE, BundleableFile.nonCompiledResourceFiles(attributes.resources())) + .addAll(BUNDLE_FILE, + Iterables.transform(compiledResources, CompiledResourceFile.TO_BUNDLED)) + .addAll(XCASSETS_DIR, + uniqueContainers(attributes.assetCatalogs(), ASSET_CATALOG_CONTAINER_TYPE)) + .addAll(ASSET_CATALOG, attributes.assetCatalogs()) + .addAll(XCDATAMODEL, datamodels); + } else { + storyboards = Storyboards.empty(); + datamodels = ImmutableList.of(); + } + + for (CompilationArtifacts artifacts : compilationArtifacts.asSet()) { + objcProvider.addAll(LIBRARY, artifacts.getArchive().asSet()); + + boolean usesCpp = false; + for (Artifact sourceFile : + Iterables.concat(artifacts.getSrcs(), artifacts.getNonArcSrcs())) { + usesCpp = usesCpp || ObjcRuleClasses.CPP_SOURCES.matches(sourceFile.getExecPath()); + } + if (usesCpp) { + objcProvider.add(FLAG, USES_CPP); + } + } + + if (alwayslink) { + for (CompilationArtifacts artifacts : compilationArtifacts.asSet()) { + for (Artifact archive : artifacts.getArchive().asSet()) { + objcProvider.add(FORCE_LOAD_LIBRARY, archive); + objcProvider.add(FORCE_LOAD_FOR_XCODEGEN, + "$(BUILT_PRODUCTS_DIR)/" + archive.getExecPath().getBaseName()); + } + } + for (Artifact archive : extraImportLibraries) { + objcProvider.add(FORCE_LOAD_LIBRARY, archive); + objcProvider.add(FORCE_LOAD_FOR_XCODEGEN, + "$(WORKSPACE_ROOT)/" + archive.getExecPath().getSafePathString()); + } + } + + objcProvider.addAll(LINKED_BINARY, linkedBinary.asSet()); + + return new ObjcCommon( + context, objcProvider.build(), storyboards, datamodels, compilationArtifacts); + } + + } + + static final FileType BUNDLE_CONTAINER_TYPE = FileType.of(".bundle"); + + static final FileType ASSET_CATALOG_CONTAINER_TYPE = FileType.of(".xcassets"); + + static final FileType FRAMEWORK_CONTAINER_TYPE = FileType.of(".framework"); + private final RuleContext context; + private final ObjcProvider objcProvider; + private final Storyboards storyboards; + private final Iterable<Xcdatamodel> datamodels; + + private final Optional<CompilationArtifacts> compilationArtifacts; + + private ObjcCommon( + RuleContext context, + ObjcProvider objcProvider, + Storyboards storyboards, + Iterable<Xcdatamodel> datamodels, + Optional<CompilationArtifacts> compilationArtifacts) { + this.context = Preconditions.checkNotNull(context); + this.objcProvider = Preconditions.checkNotNull(objcProvider); + this.storyboards = Preconditions.checkNotNull(storyboards); + this.datamodels = Preconditions.checkNotNull(datamodels); + this.compilationArtifacts = Preconditions.checkNotNull(compilationArtifacts); + } + + public ObjcProvider getObjcProvider() { + return objcProvider; + } + + public Optional<CompilationArtifacts> getCompilationArtifacts() { + return compilationArtifacts; + } + + /** + * Returns all storyboards declared in this rule (not including others in the transitive + * dependency tree). + */ + public Storyboards getStoryboards() { + return storyboards; + } + + /** + * Returns all datamodels declared in this rule (not including others in the transitive + * dependency tree). + */ + public Iterable<Xcdatamodel> getDatamodels() { + return datamodels; + } + + /** + * Returns an {@link Optional} containing the compiled {@code .a} file, or + * {@link Optional#absent()} if this object contains no {@link CompilationArtifacts} or the + * compilation information has no sources. + */ + public Optional<Artifact> getCompiledArchive() { + for (CompilationArtifacts justCompilationArtifacts : compilationArtifacts.asSet()) { + return justCompilationArtifacts.getArchive(); + } + return Optional.absent(); + } + + /** + * Reports any known errors to the {@link RuleContext}. This should be called exactly once for + * a target. + */ + public void reportErrors() { + + // TODO(bazel-team): Report errors for rules that are not actually useful (i.e. objc_library + // without sources or resources, empty objc_bundles) + } + + static ImmutableList<PathFragment> userHeaderSearchPaths(BuildConfiguration configuration) { + return ImmutableList.of( + new PathFragment("."), + configuration.getGenfilesFragment()); + } + + /** + * Returns the first directory in the sequence of parents of the exec path of the given artifact + * that matches {@code type}. For instance, if {@code type} is FileType.of(".foo") and the exec + * path of {@code artifact} is {@code a/b/c/bar.foo/d/e}, then the return value is + * {@code a/b/c/bar.foo}. + */ + static Optional<PathFragment> nearestContainerMatching(FileType type, Artifact artifact) { + PathFragment container = artifact.getExecPath(); + do { + if (type.matches(container)) { + return Optional.of(container); + } + container = container.getParentDirectory(); + } while (container != null); + return Optional.absent(); + } + + /** + * Similar to {@link #nearestContainerMatching(FileType, Artifact)}, but tries matching several + * file types in {@code types}, and returns a path for the first match in the sequence. + */ + static Optional<PathFragment> nearestContainerMatching( + Iterable<FileType> types, Artifact artifact) { + for (FileType type : types) { + for (PathFragment container : nearestContainerMatching(type, artifact).asSet()) { + return Optional.of(container); + } + } + return Optional.absent(); + } + + /** + * Returns all directories matching {@code containerType} that contain the items in + * {@code artifacts}. This function ignores artifacts that are not in any directory matching + * {@code containerType}. + */ + static Iterable<PathFragment> uniqueContainers( + Iterable<Artifact> artifacts, FileType containerType) { + ImmutableSet.Builder<PathFragment> containers = new ImmutableSet.Builder<>(); + for (Artifact artifact : artifacts) { + containers.addAll(ObjcCommon.nearestContainerMatching(containerType, artifact).asSet()); + } + return containers.build(); + } + + /** + * Similar to {@link #nearestContainerMatching(FileType, Artifact)}, but returns the container + * closest to the root that matches the given type. + */ + static Optional<PathFragment> farthestContainerMatching(FileType type, Artifact artifact) { + PathFragment container = artifact.getExecPath(); + Optional<PathFragment> lastMatch = Optional.absent(); + do { + if (type.matches(container)) { + lastMatch = Optional.of(container); + } + container = container.getParentDirectory(); + } while (container != null); + return lastMatch; + } + + static Iterable<String> notInContainerErrors( + Iterable<Artifact> artifacts, FileType containerType) { + return notInContainerErrors(artifacts, ImmutableList.of(containerType)); + } + + @VisibleForTesting + static final String NOT_IN_CONTAINER_ERROR_FORMAT = + "File '%s' is not in a directory of one of these type(s): %s"; + + static Iterable<String> notInContainerErrors( + Iterable<Artifact> artifacts, Iterable<FileType> containerTypes) { + Set<String> errors = new HashSet<>(); + for (Artifact artifact : artifacts) { + boolean inContainer = nearestContainerMatching(containerTypes, artifact).isPresent(); + if (!inContainer) { + errors.add(String.format(NOT_IN_CONTAINER_ERROR_FORMAT, + artifact.getExecPath(), Iterables.toString(containerTypes))); + } + } + return errors; + } + + /** + * @param filesToBuild files to build for this target. These also become the data runfiles. Note + * that this method may add more files to create the complete list of files to build for this + * target. + * @param maybeTargetProvider the provider for this target. + * @param maybeExportedProvider the {@link ObjcProvider} for this target. This should generally be + * present whenever {@code objc_} rules may depend on this target. + * @param maybeJ2ObjcSrcsProvider the {@link J2ObjcSrcsProvider} for this target. + */ + public ConfiguredTarget configuredTarget(NestedSet<Artifact> filesToBuild, + Optional<XcodeProvider> maybeTargetProvider, Optional<ObjcProvider> maybeExportedProvider, + Optional<XcTestAppProvider> maybeXcTestAppProvider, + Optional<J2ObjcSrcsProvider> maybeJ2ObjcSrcsProvider) { + NestedSet<Artifact> allFilesToBuild = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(filesToBuild) + .addTransitive(storyboards.getOutputZips()) + .addAll(Xcdatamodel.outputZips(datamodels)) + .build(); + + RunfilesProvider runfilesProvider = RunfilesProvider.withData( + new Runfiles.Builder() + .addRunfiles(context, RunfilesProvider.DEFAULT_RUNFILES) + .build(), + new Runfiles.Builder().addTransitiveArtifacts(allFilesToBuild).build()); + + RuleConfiguredTargetBuilder target = new RuleConfiguredTargetBuilder(context) + .setFilesToBuild(allFilesToBuild) + .add(RunfilesProvider.class, runfilesProvider); + for (ObjcProvider exportedProvider : maybeExportedProvider.asSet()) { + target.addProvider(ObjcProvider.class, exportedProvider); + } + for (XcTestAppProvider xcTestAppProvider : maybeXcTestAppProvider.asSet()) { + target.addProvider(XcTestAppProvider.class, xcTestAppProvider); + } + for (XcodeProvider targetProvider : maybeTargetProvider.asSet()) { + target.addProvider(XcodeProvider.class, targetProvider); + } + for (J2ObjcSrcsProvider j2ObjcSrcsProvider : maybeJ2ObjcSrcsProvider.asSet()) { + target.addProvider(J2ObjcSrcsProvider.class, j2ObjcSrcsProvider); + } + return target.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfiguration.java new file mode 100644 index 0000000000..3f2e073dd0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfiguration.java @@ -0,0 +1,136 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.CompilationMode; +import com.google.devtools.build.xcode.common.Platform; + +import java.util.List; + +/** + * A compiler configuration containing flags required for Objective-C compilation. + */ +public class ObjcConfiguration extends BuildConfiguration.Fragment { + @VisibleForTesting + static final ImmutableList<String> DBG_COPTS = ImmutableList.of("-O0", "-DDEBUG=1", + "-fstack-protector", "-fstack-protector-all", "-D_GLIBCXX_DEBUG_PEDANTIC", "-D_GLIBCXX_DEBUG", + "-D_GLIBCPP_CONCEPT_CHECKS"); + + @VisibleForTesting + static final ImmutableList<String> FASTBUILD_COPTS = ImmutableList.of("-O0", "-DDEBUG=1"); + + @VisibleForTesting + static final ImmutableList<String> OPT_COPTS = + ImmutableList.of("-Os", "-DNDEBUG=1", "-Wno-unused-variable", "-Winit-self", "-Wno-extra"); + + private final String iosSdkVersion; + private final String iosMinimumOs; + private final String iosSimulatorVersion; + private final String iosCpu; + private final String xcodeOptions; + private final boolean generateDebugSymbols; + private final List<String> copts; + private final CompilationMode compilationMode; + + ObjcConfiguration(ObjcCommandLineOptions objcOptions, BuildConfiguration.Options options) { + this.iosSdkVersion = Preconditions.checkNotNull(objcOptions.iosSdkVersion, "iosSdkVersion"); + this.iosMinimumOs = Preconditions.checkNotNull(objcOptions.iosMinimumOs, "iosMinimumOs"); + this.iosSimulatorVersion = + Preconditions.checkNotNull(objcOptions.iosSimulatorVersion, "iosSimulatorVersion"); + this.iosCpu = Preconditions.checkNotNull(objcOptions.iosCpu, "iosCpu"); + this.xcodeOptions = Preconditions.checkNotNull(objcOptions.xcodeOptions, "xcodeOptions"); + this.generateDebugSymbols = objcOptions.generateDebugSymbols; + this.copts = ImmutableList.copyOf(objcOptions.copts); + this.compilationMode = Preconditions.checkNotNull(options.compilationMode, "compilationMode"); + } + + public String getIosSdkVersion() { + return iosSdkVersion; + } + + /** + * Returns the minimum iOS version supported by binaries and libraries. Any dependencies on newer + * iOS version features or libraries will become weak dependencies which are only loaded if the + * runtime OS supports them. + */ + public String getMinimumOs() { + return iosMinimumOs; + } + + public String getIosSimulatorVersion() { + return iosSimulatorVersion; + } + + public String getIosCpu() { + return iosCpu; + } + + public Platform getPlatform() { + return Platform.forArch(getIosCpu()); + } + + public String getXcodeOptions() { + return xcodeOptions; + } + + public boolean generateDebugSymbols() { + return generateDebugSymbols; + } + + /** + * Returns the current compilation mode. + */ + public CompilationMode getCompilationMode() { + return compilationMode; + } + + /** + * Returns the default set of clang options for the current compilation mode. + */ + public List<String> getCoptsForCompilationMode() { + switch (compilationMode) { + case DBG: + return DBG_COPTS; + case FASTBUILD: + return FASTBUILD_COPTS; + case OPT: + return OPT_COPTS; + default: + throw new AssertionError(); + } + } + + /** + * Returns options passed to (Apple) clang when compiling Objective C. These options should be + * applied after any default options but before options specified in the attributes of the rule. + */ + public List<String> getCopts() { + return copts; + } + + @Override + public String getName() { + return "Objective-C"; + } + + @Override + public String cacheKey() { + return iosSdkVersion; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfigurationLoader.java new file mode 100644 index 0000000000..19713a346a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfigurationLoader.java @@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; + +/** + * A loader that creates ObjcConfiguration instances based on Objective-C configurations and + * command-line options. + */ +public class ObjcConfigurationLoader implements ConfigurationFragmentFactory { + @Override + public ObjcConfiguration create(ConfigurationEnvironment env, BuildOptions buildOptions) + throws InvalidConfigurationException { + return new ObjcConfiguration(buildOptions.get(ObjcCommandLineOptions.class), + buildOptions.get(BuildConfiguration.Options.class)); + } + + @Override + public Class<? extends BuildConfiguration.Fragment> creates() { + return ObjcConfiguration.class; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFramework.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFramework.java new file mode 100644 index 0000000000..c6a003757e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFramework.java @@ -0,0 +1,60 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.collect.nestedset.Order.STABLE_ORDER; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.objc.ObjcSdkFrameworks.Attributes; + +/** + * Implementation for the {@code objc_framework} rule. + */ +public class ObjcFramework implements RuleConfiguredTargetFactory { + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + Attributes sdkFrameworkAttributes = new Attributes(ruleContext); + + ImmutableList<Artifact> frameworkImports = + ruleContext.getPrerequisiteArtifacts("framework_imports", Mode.TARGET).list(); + ObjcCommon common = new ObjcCommon.Builder(ruleContext) + .addFrameworkImports( + frameworkImports) + .addExtraSdkFrameworks(sdkFrameworkAttributes.sdkFrameworks()) + .addExtraWeakSdkFrameworks(sdkFrameworkAttributes.weakSdkFrameworks()) + .addExtraSdkDylibs(sdkFrameworkAttributes.sdkDylibs()) + .build(); + + Iterable<String> containerErrors = + ObjcCommon.notInContainerErrors(frameworkImports, ObjcCommon.FRAMEWORK_CONTAINER_TYPE); + for (String error : containerErrors) { + ruleContext.attributeError("framework_imports", error); + } + + return common.configuredTarget( + NestedSetBuilder.<Artifact>emptySet(STABLE_ORDER) /* filesToBuild */, + Optional.<XcodeProvider>absent(), + Optional.of(common.getObjcProvider()), + Optional.<XcTestAppProvider>absent(), + Optional.<J2ObjcSrcsProvider>absent()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFrameworkRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFrameworkRule.java new file mode 100644 index 0000000000..7fcfdd379f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFrameworkRule.java @@ -0,0 +1,62 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcSdkFrameworksRule; +import com.google.devtools.build.lib.util.FileTypeSet; + +/** + * Rule definition for objc_framework. + */ +@BlazeRule(name = "objc_framework", + factoryClass = ObjcFramework.class, + ancestors = { BaseRuleClasses.BaseRule.class, ObjcSdkFrameworksRule.class}) +public class ObjcFrameworkRule implements RuleDefinition { + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE(objc_framework).ATTRIBUTE(framework_imports) --> + The list of files under a <code>.framework</code> directory which are + provided to Objective-C targets that depend on this target. + <i>(List of <a href="build-ref.html#labels">labels</a>; required)</i> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("framework_imports", LABEL_LIST) + .allowedFileTypes(FileTypeSet.ANY_FILE) + .mandatory() + .nonEmpty()) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_framework, TYPE = LIBRARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule encapsulates an already-built framework. It is defined by a list +of files in one or more <code>.framework</code> directories. + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/ diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImport.java new file mode 100644 index 0000000000..70743ed797 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImport.java @@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.XcodeProductType.LIBRARY_STATIC; + +import com.google.common.base.Optional; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.CompilationAttributes; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.ResourceAttributes; + +/** + * Implementation for {@code objc_import}. + */ +public class ObjcImport implements RuleConfiguredTargetFactory { + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + ObjcCommon common = new ObjcCommon.Builder(ruleContext) + .setCompilationAttributes(new CompilationAttributes(ruleContext)) + .setResourceAttributes(new ResourceAttributes(ruleContext)) + .setIntermediateArtifacts(ObjcRuleClasses.intermediateArtifacts(ruleContext)) + .setAlwayslink(ruleContext.attributes().get("alwayslink", Type.BOOLEAN)) + .addExtraImportLibraries( + ruleContext.getPrerequisiteArtifacts("archives", Mode.TARGET).list()) + .build(); + + XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder(); + NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.stableOrder(); + + new CompilationSupport(ruleContext) + .addXcodeSettings(xcodeProviderBuilder, common, OptionsProvider.DEFAULT) + .validateAttributes(); + + new ResourceSupport(ruleContext) + .registerActions(common.getStoryboards()) + .validateAttributes() + .addXcodeSettings(xcodeProviderBuilder); + + new XcodeSupport(ruleContext) + .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), LIBRARY_STATIC) + .registerActions(xcodeProviderBuilder.build()) + .addFilesToBuild(filesToBuild); + + return common.configuredTarget( + filesToBuild.build(), + Optional.of(xcodeProviderBuilder.build()), + Optional.of(common.getObjcProvider()), + Optional.<XcTestAppProvider>absent(), + Optional.<J2ObjcSrcsProvider>absent()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImportRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImportRule.java new file mode 100644 index 0000000000..24e9412c5a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImportRule.java @@ -0,0 +1,81 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcCompilationRule; +import com.google.devtools.build.lib.util.FileType; + +/** + * Rule definition for {@code objc_import}. + */ +@BlazeRule(name = "objc_import", + factoryClass = ObjcImport.class, + ancestors = { ObjcCompilationRule.class }) +public class ObjcImportRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /*<!-- #BLAZE_RULE(objc_import).IMPLICIT_OUTPUTS --> + <ul> + <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: An Xcode project file which + can be used to develop or build on a Mac.</li> + </ul> + <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/ + .setImplicitOutputsFunction(XcodeSupport.PBXPROJ) + /* <!-- #BLAZE_RULE(objc_import).ATTRIBUTE(archives) --> + The list of <code>.a</code> files provided to Objective-C targets that + depend on this target. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("archives", LABEL_LIST) + .mandatory() + .nonEmpty() + .allowedFileTypes(FileType.of(".a"))) + /* <!-- #BLAZE_RULE(objc_import).ATTRIBUTE(alwayslink) --> + If 1, any bundle or binary that depends (directly or indirectly) on this + library will link in all the archive files listed in + <code>archives</code>, even if some contain no symbols referenced by the + binary. + ${SYNOPSIS} + This is useful if your code isn't explicitly called by code in + the binary, e.g., if your code registers to receive some callback + provided by some service. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("alwayslink", BOOLEAN)) + .removeAttribute("deps") + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_import, TYPE = LIBRARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule encapsulates an already-compiled static library in the form of an +<code>.a</code> file. It also allows exporting headers and resources using the same +attributes supported by <code>objc_library</code>.</p> + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/ diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibrary.java new file mode 100644 index 0000000000..4136ffefce --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibrary.java @@ -0,0 +1,132 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.XcodeProductType.LIBRARY_STATIC; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.CompilationAttributes; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.ResourceAttributes; + +/** + * Implementation for {@code objc_library}. + */ +public class ObjcLibrary implements RuleConfiguredTargetFactory { + + /** + * An {@link IterableWrapper} containing extra library {@link Artifact}s to be linked into the + * final ObjC application bundle. + */ + static final class ExtraImportLibraries extends IterableWrapper<Artifact> { + ExtraImportLibraries(Artifact... extraImportLibraries) { + super(extraImportLibraries); + } + } + + /** + * An {@link IterableWrapper} containing defines as specified in the {@code defines} attribute to + * be applied to this target and all depending targets' compilation actions. + */ + static final class Defines extends IterableWrapper<String> { + Defines(Iterable<String> defines) { + super(defines); + } + + Defines(String... defines) { + super(defines); + } + } + + /** + * Constructs an {@link ObjcCommon} instance based on the attributes of the given rule. The rule + * should inherit from {@link ObjcLibraryRule}.. + */ + static ObjcCommon common(RuleContext ruleContext, Iterable<SdkFramework> extraSdkFrameworks, + boolean alwayslink, ExtraImportLibraries extraImportLibraries, Defines defines, + Iterable<ObjcProvider> extraDepObjcProviders) { + CompilationArtifacts compilationArtifacts = + CompilationSupport.compilationArtifacts(ruleContext); + + return new ObjcCommon.Builder(ruleContext) + .setCompilationAttributes(new CompilationAttributes(ruleContext)) + .setResourceAttributes(new ResourceAttributes(ruleContext)) + .addExtraSdkFrameworks(extraSdkFrameworks) + .addDefines(defines) + .setCompilationArtifacts(compilationArtifacts) + .addDepObjcProviders(ruleContext.getPrerequisites("deps", Mode.TARGET, ObjcProvider.class)) + .addDepObjcProviders( + ruleContext.getPrerequisites("bundles", Mode.TARGET, ObjcProvider.class)) + .addNonPropagatedDepObjcProviders(ruleContext.getPrerequisites("non_propagated_deps", + Mode.TARGET, ObjcProvider.class)) + .setIntermediateArtifacts(ObjcRuleClasses.intermediateArtifacts(ruleContext)) + .setAlwayslink(alwayslink) + .addExtraImportLibraries(extraImportLibraries) + .addDepObjcProviders(extraDepObjcProviders) + .build(); + } + + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + ObjcCommon common = common( + ruleContext, ImmutableList.<SdkFramework>of(), + ruleContext.attributes().get("alwayslink", Type.BOOLEAN), new ExtraImportLibraries(), + new Defines(ruleContext.getTokenizedStringListAttr("defines")), + ImmutableList.<ObjcProvider>of()); + OptionsProvider optionsProvider = optionsProvider(ruleContext); + + XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder(); + NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.<Artifact>stableOrder() + .addAll(common.getCompiledArchive().asSet()); + + new CompilationSupport(ruleContext) + .registerCompileAndArchiveActions(common, optionsProvider) + .addXcodeSettings(xcodeProviderBuilder, common, optionsProvider) + .validateAttributes(); + + new ResourceSupport(ruleContext) + .registerActions(common.getStoryboards()) + .validateAttributes() + .addXcodeSettings(xcodeProviderBuilder); + + new XcodeSupport(ruleContext) + .addFilesToBuild(filesToBuild) + .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), LIBRARY_STATIC) + .addDependencies(xcodeProviderBuilder) + .registerActions(xcodeProviderBuilder.build()); + + return common.configuredTarget( + filesToBuild.build(), + Optional.of(xcodeProviderBuilder.build()), + Optional.of(common.getObjcProvider()), + Optional.<XcTestAppProvider>absent(), + Optional.of(ObjcRuleClasses.j2ObjcSrcsProvider(ruleContext))); + } + + private OptionsProvider optionsProvider(RuleContext ruleContext) { + return new OptionsProvider.Builder() + .addCopts(ruleContext.getTokenizedStringListAttr("copts")) + .addTransitive(Optional.fromNullable( + ruleContext.getPrerequisite("options", Mode.TARGET, OptionsProvider.class))) + .build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryRule.java new file mode 100644 index 0000000000..f721492bcf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryRule.java @@ -0,0 +1,157 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; +import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.NON_ARC_SRCS_TYPE; +import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.SRCS_TYPE; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcCompilationRule; +import com.google.devtools.build.lib.util.FileType; + +/** + * Rule definition for objc_library. + */ +@BlazeRule(name = "objc_library", + factoryClass = ObjcLibrary.class, + ancestors = { ObjcCompilationRule.class, + ObjcRuleClasses.ObjcOptsRule.class }) +public class ObjcLibraryRule implements RuleDefinition { + private static final Iterable<String> ALLOWED_DEPS_RULE_CLASSES = ImmutableSet.of( + "objc_library", + "objc_import", + "objc_bundle", + "objc_framework", + "objc_bundle_library", + "objc_proto_library", + "j2objc_library"); + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /*<!-- #BLAZE_RULE(objc_library).IMPLICIT_OUTPUTS --> + <ul> + <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: An Xcode project file which + can be used to develop or build on a Mac.</li> + </ul> + <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/ + .setImplicitOutputsFunction(XcodeSupport.PBXPROJ) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(srcs) --> + The list of C, C++, Objective-C, and Objective-C++ files that are + processed to create the library target. + ${SYNOPSIS} + These are your checked-in source files, plus any generated files. + These are compiled into .o files with Clang, so headers should not go + here (see the hdrs attribute). + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("srcs", LABEL_LIST) + .direct_compile_time_input() + .allowedFileTypes(SRCS_TYPE)) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(non_arc_srcs) --> + The list of Objective-C files that are processed to create the + library target that DO NOT use ARC. + ${SYNOPSIS} + The files in this attribute are treated very similar to those in the + srcs attribute, but are compiled without ARC enabled. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("non_arc_srcs", LABEL_LIST) + .direct_compile_time_input() + .allowedFileTypes(NON_ARC_SRCS_TYPE)) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(pch) --> + Header file to prepend to every source file being compiled (both arc + and non-arc). Note that the file will not be precompiled - this is + simply a convenience, not a build-speed enhancement. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("pch", LABEL) + .direct_compile_time_input() + .allowedFileTypes(FileType.of(".pch"))) + .add(attr("options", LABEL) + .undocumented("objc_options will probably be removed") + .allowedFileTypes() + .allowedRuleClasses("objc_options")) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(alwayslink) --> + If 1, any bundle or binary that depends (directly or indirectly) on this + library will link in all the object files for the files listed in + <code>srcs</code> and <code>non_arc_srcs</code>, even if some contain no + symbols referenced by the binary. + ${SYNOPSIS} + This is useful if your code isn't explicitly called by code in + the binary, e.g., if your code registers to receive some callback + provided by some service. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("alwayslink", BOOLEAN)) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(deps) --> + The list of targets that are linked together to form the final bundle. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .override(attr("deps", LABEL_LIST) + .direct_compile_time_input() + .allowedRuleClasses(ALLOWED_DEPS_RULE_CLASSES) + .allowedFileTypes()) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(bundles) --> + The list of bundle targets that this target requires to be included in the final bundle. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("bundles", LABEL_LIST) + .direct_compile_time_input() + .allowedRuleClasses("objc_bundle", "objc_bundle_library") + .allowedFileTypes()) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(non_propagated_deps) --> + The list of targets that are required in order to build this target, + but which are not included in the final bundle. + <br /> + This attribute should only rarely be used, and probably only for proto + dependencies. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("non_propagated_deps", LABEL_LIST) + .direct_compile_time_input() + .allowedRuleClasses(ALLOWED_DEPS_RULE_CLASSES) + .allowedFileTypes()) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(defines) --> + Extra <code>-D</code> flags to pass to the compiler. They should be in + the form <code>KEY=VALUE</code> or simply <code>KEY</code> and are + passed not only the compiler for this target (as <code>copts</code> + are) but also to all <code>objc_</code> dependers of this target. + ${SYNOPSIS} + Subject to <a href="#make_variables">"Make variable"</a> substitution and + <a href="#sh-tokenization">Bourne shell tokenization</a>. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("defines", STRING_LIST)) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_library, TYPE = LIBRARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule produces a static library from the given Objective-C source files.</p> + +${IMPLICIT_OUTPUTS} + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/ diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptions.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptions.java new file mode 100644 index 0000000000..a7e2b8f60f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptions.java @@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * Implementation for the {@code objc_options} rule. + */ +public class ObjcOptions implements RuleConfiguredTargetFactory { + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + return new RuleConfiguredTargetBuilder(ruleContext) + .add(RunfilesProvider.class, RunfilesProvider.EMPTY) + .add(OptionsProvider.class, + new OptionsProvider.Builder() + .addCopts(ruleContext.getTokenizedStringListAttr("copts")) + .addInfoplists( + ruleContext.getPrerequisiteArtifacts("infoplists", Mode.TARGET).list()) + .build()) + .build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptionsRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptionsRule.java new file mode 100644 index 0000000000..7f26bfef87 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptionsRule.java @@ -0,0 +1,67 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.PLIST_TYPE; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses.BaseRule; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcOptsRule; + +/** + * Rule definition for {@code objc_options}. + */ +@BlazeRule(name = "objc_options", + factoryClass = ObjcOptions.class, + ancestors = { BaseRule.class, ObjcOptsRule.class }) +public class ObjcOptionsRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + // TODO(bazel-team): Figure out if we really need objc_options, and if + // we don't, delete it. + .setUndocumented() + /* <!-- #BLAZE_RULE(objc_options).ATTRIBUTE(xcode_name)[DEPRECATED] --> + This attribute is ignored and will be removed. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("xcode_name", Type.STRING)) + /* <!-- #BLAZE_RULE(objc_options).ATTRIBUTE(infoplists) --> + infoplist files to merge with the final binary's infoplist. This + corresponds to a single file <i>appname</i>-Info.plist in Xcode + projects. + <i>(List of <a href="build-ref.html#labels">labels</a>; optional)</i> + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("infoplists", Type.LABEL_LIST) + .allowedFileTypes(PLIST_TYPE)) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_options, TYPE = OTHER, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule provides a nameable set of build settings to use when building +Objective-C targets.</p> + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/ diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibrary.java new file mode 100644 index 0000000000..647b221258 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibrary.java @@ -0,0 +1,242 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.common.base.CaseFormat.LOWER_UNDERSCORE; +import static com.google.common.base.CaseFormat.UPPER_CAMEL; +import static com.google.devtools.build.lib.rules.objc.XcodeProductType.LIBRARY_STATIC; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.proto.ProtoSourcesProvider; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import javax.annotation.Nullable; + +/** + * Implementation for the "objc_proto_library" rule. + */ +public class ObjcProtoLibrary implements RuleConfiguredTargetFactory { + private static final Function<Artifact, PathFragment> PARENT_PATHFRAGMENT = + new Function<Artifact, PathFragment>() { + @Override + public PathFragment apply(Artifact input) { + return input.getExecPath().getParentDirectory(); + } + }; + + @VisibleForTesting + static final String NO_PROTOS_ERROR = + "no protos to compile - a non-empty deps attribute is required"; + + @Override + public ConfiguredTarget create(final RuleContext ruleContext) throws InterruptedException { + Artifact compileProtos = ruleContext.getPrerequisiteArtifact( + ObjcRuleClasses.ObjcProtoRule.COMPILE_PROTOS_ATTR, Mode.HOST); + Optional<Artifact> optionsFile = Optional.fromNullable( + ruleContext.getPrerequisiteArtifact(ObjcProtoLibraryRule.OPTIONS_FILE_ATTR, Mode.HOST)); + NestedSet<Artifact> protos = NestedSetBuilder.<Artifact>stableOrder() + .addAll(ruleContext.getPrerequisiteArtifacts("deps", Mode.TARGET) + .filter(FileType.of(".proto")) + .list()) + .addTransitive(maybeGetProtoSources(ruleContext)) + .build(); + + if (Iterables.isEmpty(protos)) { + ruleContext.ruleError(NO_PROTOS_ERROR); + } + + ImmutableList<Artifact> libProtobuf = ruleContext + .getPrerequisiteArtifacts(ObjcProtoLibraryRule.LIBPROTOBUF_ATTR, Mode.TARGET) + .list(); + ImmutableList<Artifact> protoSupport = ruleContext + .getPrerequisiteArtifacts(ObjcRuleClasses.ObjcProtoRule.PROTO_SUPPORT_ATTR, Mode.HOST) + .list(); + + // Generate sources in a package-and-rule-scoped directory; adds both the + // package-and-rule-scoped directory and the header-containing-directory to the include path of + // dependers. + PathFragment rootRelativeOutputDir = new PathFragment( + ruleContext.getLabel().getPackageFragment(), + new PathFragment("_generated_protos_" + ruleContext.getLabel().getName())); + PathFragment workspaceRelativeOutputDir = new PathFragment( + ruleContext.getBinOrGenfilesDirectory().getExecPath(), rootRelativeOutputDir); + PathFragment generatedProtoDir = + new PathFragment(workspaceRelativeOutputDir, ruleContext.getLabel().getPackageFragment()); + + boolean outputCpp = + ruleContext.attributes().get(ObjcProtoLibraryRule.OUTPUT_CPP_ATTR, Type.BOOLEAN); + + ImmutableList<Artifact> protoGeneratedSources = outputArtifacts( + ruleContext, rootRelativeOutputDir, protos, FileType.of(".pb." + (outputCpp ? "cc" : "m")), + outputCpp); + ImmutableList<Artifact> protoGeneratedHeaders = outputArtifacts( + ruleContext, rootRelativeOutputDir, protos, FileType.of(".pb.h"), outputCpp); + + Artifact inputFileList = ruleContext.getAnalysisEnvironment().getDerivedArtifact( + AnalysisUtils.getUniqueDirectory(ruleContext.getLabel(), new PathFragment("_protos")) + .getRelative("_proto_input_files"), + ruleContext.getConfiguration().getGenfilesDirectory()); + + ruleContext.registerAction(new FileWriteAction( + ruleContext.getActionOwner(), + inputFileList, + ObjcActionsBuilder.joinExecPaths(protos), + false)); + + CustomCommandLine.Builder commandLineBuilder = new CustomCommandLine.Builder() + .add(compileProtos.getExecPathString()) + .add("--input-file-list").add(inputFileList.getExecPathString()) + .add("--output-dir").add(workspaceRelativeOutputDir.getSafePathString()); + if (optionsFile.isPresent()) { + commandLineBuilder + .add("--compiler-options-path") + .add(optionsFile.get().getExecPathString()); + } + if (outputCpp) { + commandLineBuilder.add("--generate-cpp"); + } + + if (!Iterables.isEmpty(protos)) { + ruleContext.registerAction(new SpawnAction.Builder() + .setMnemonic("GenObjcProtos") + .addInput(compileProtos) + .addInputs(optionsFile.asSet()) + .addInputs(protos) + .addInput(inputFileList) + .addInputs(libProtobuf) + .addInputs(protoSupport) + .addOutputs(Iterables.concat(protoGeneratedSources, protoGeneratedHeaders)) + .setExecutable(new PathFragment("/usr/bin/python")) + .setCommandLine(commandLineBuilder.build()) + .setExecutionInfo(ImmutableMap.of(ExecutionRequirements.REQUIRES_DARWIN, "")) + .build(ruleContext)); + } + + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + CompilationArtifacts compilationArtifacts = new CompilationArtifacts.Builder() + .addNonArcSrcs(protoGeneratedSources) + .setIntermediateArtifacts(intermediateArtifacts) + .setPchFile(Optional.<Artifact>absent()) + .build(); + + ImmutableSet<PathFragment> searchPathEntries = new ImmutableSet.Builder<PathFragment>() + .add(workspaceRelativeOutputDir) + .add(generatedProtoDir) + .addAll(Iterables.transform(protoGeneratedHeaders, PARENT_PATHFRAGMENT)) + .build(); + ObjcCommon common = new ObjcCommon.Builder(ruleContext) + .setCompilationArtifacts(compilationArtifacts) + .addUserHeaderSearchPaths(searchPathEntries) + .addDepObjcProviders(ruleContext.getPrerequisites( + ObjcProtoLibraryRule.LIBPROTOBUF_ATTR, Mode.TARGET, ObjcProvider.class)) + .setIntermediateArtifacts(intermediateArtifacts) + .addHeaders(protoGeneratedHeaders) + .addHeaders(protoGeneratedSources) + .build(); + + XcodeProvider xcodeProvider = new XcodeProvider.Builder() + .setLabel(ruleContext.getLabel()) + .addUserHeaderSearchPaths(searchPathEntries) + .addDependencies(ruleContext.getPrerequisites( + ObjcProtoLibraryRule.LIBPROTOBUF_ATTR, Mode.TARGET, XcodeProvider.class)) + .addCopts(ObjcRuleClasses.objcConfiguration(ruleContext).getCopts()) + .setProductType(LIBRARY_STATIC) + .addHeaders(protoGeneratedHeaders) + .setCompilationArtifacts(common.getCompilationArtifacts().get()) + .setObjcProvider(common.getObjcProvider()) + .build(); + + ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext); + actionsBuilder + .registerCompileAndArchiveActions( + compilationArtifacts, common.getObjcProvider(), OptionsProvider.DEFAULT); + actionsBuilder.registerXcodegenActions( + new ObjcRuleClasses.Tools(ruleContext), + ruleContext.getImplicitOutputArtifact(XcodeSupport.PBXPROJ), + XcodeProvider.Project.fromTopLevelTarget(xcodeProvider)); + + return common.configuredTarget( + NestedSetBuilder.<Artifact>stableOrder() + .addAll(common.getCompiledArchive().asSet()) + .addAll(protoGeneratedSources) + .addAll(protoGeneratedHeaders) + .add(ruleContext.getImplicitOutputArtifact(XcodeSupport.PBXPROJ)) + .build(), + Optional.of(xcodeProvider), + Optional.of(common.getObjcProvider()), + Optional.<XcTestAppProvider>absent(), + Optional.<J2ObjcSrcsProvider>absent()); + } + + private NestedSet<Artifact> maybeGetProtoSources(RuleContext ruleContext) { + NestedSetBuilder<Artifact> artifacts = new NestedSetBuilder<>(Order.STABLE_ORDER); + Iterable<ProtoSourcesProvider> providers = + ruleContext.getPrerequisites("deps", Mode.TARGET, ProtoSourcesProvider.class); + for (ProtoSourcesProvider provider : providers) { + artifacts.addTransitive(provider.getTransitiveProtoSources()); + } + return artifacts.build(); + } + + private ImmutableList<Artifact> outputArtifacts(RuleContext ruleContext, + PathFragment rootRelativeOutputDir, Iterable<Artifact> protos, FileType newFileType, + boolean outputCpp) { + ImmutableList.Builder<Artifact> builder = new ImmutableList.Builder<>(); + for (Artifact proto : protos) { + String protoOutputName; + if (outputCpp) { + protoOutputName = proto.getFilename(); + } else { + String lowerUnderscoreBaseName = proto.getFilename().replace('-', '_').toLowerCase(); + protoOutputName = LOWER_UNDERSCORE.to(UPPER_CAMEL, lowerUnderscoreBaseName); + } + PathFragment rawFragment = new PathFragment( + rootRelativeOutputDir, + proto.getExecPath().getParentDirectory(), + new PathFragment(protoOutputName)); + @Nullable PathFragment outputFile = FileSystemUtils.replaceExtension( + rawFragment, + newFileType.getExtensions().get(0), + ".proto"); + if (outputFile != null) { + builder.add(ruleContext.getAnalysisEnvironment().getDerivedArtifact( + outputFile, ruleContext.getBinOrGenfilesDirectory())); + } + } + return builder.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibraryRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibraryRule.java new file mode 100644 index 0000000000..a25f96e6fa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibraryRule.java @@ -0,0 +1,80 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for objc_proto_library. + * + * This is a temporary rule until it is better known how to support proto_library rules. + */ +@BlazeRule(name = "objc_proto_library", + factoryClass = ObjcProtoLibrary.class, + ancestors = { BaseRuleClasses.RuleBase.class, ObjcRuleClasses.ObjcProtoRule.class }) +public class ObjcProtoLibraryRule implements RuleDefinition { + static final String OPTIONS_FILE_ATTR = "options_file"; + static final String OUTPUT_CPP_ATTR = "output_cpp"; + static final String LIBPROTOBUF_ATTR = "$lib_protobuf"; + + @Override + public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE(objc_proto_library).ATTRIBUTE(deps) --> + The directly depended upon proto_library rules. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .override(attr("deps", LABEL_LIST) + .allowedRuleClasses("proto_library", "filegroup") + .legacyAllowAnyFileType()) + /* <!-- #BLAZE_RULE(objc_proto_library).ATTRIBUTE(options_file) --> + Optional options file to apply to protos which affects compilation (e.g. class + whitelist/blacklist settings). + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr(OPTIONS_FILE_ATTR, LABEL).legacyAllowAnyFileType().singleArtifact().cfg(HOST)) + /* <!-- #BLAZE_RULE(objc_proto_library).ATTRIBUTE(output_cpp) --> + If true, output C++ rather than ObjC. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr(OUTPUT_CPP_ATTR, BOOLEAN).value(false)) + // TODO(bazel-team): Use //external:objc_proto_lib when bind() support is a little better + .add(attr(LIBPROTOBUF_ATTR, LABEL).allowedRuleClasses("objc_library") + .value(env.getLabel( + "//googlemac/ThirdParty/ProtocolBuffers2/objectivec:ProtocolBuffers_lib"))) + .add(attr("$xcodegen", LABEL).cfg(HOST).exec() + .value(env.getLabel("//tools/objc:xcodegen"))) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_proto_library, TYPE = LIBRARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule produces a static library from the given proto_library dependencies, after applying an +options file.</p> + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/ diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProvider.java new file mode 100644 index 0000000000..c48710ed16 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProvider.java @@ -0,0 +1,313 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.collect.nestedset.Order.LINK_ORDER; +import static com.google.devtools.build.lib.collect.nestedset.Order.STABLE_ORDER; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.TargetControl; + +import java.util.HashMap; +import java.util.Map; + +/** + * A provider that provides all compiling and linking information in the transitive closure of its + * deps that are needed for building Objective-C rules. + */ +@Immutable +public final class ObjcProvider implements TransitiveInfoProvider { + /** + * Represents one of the things this provider can provide transitively. Things are provided as + * {@link NestedSet}s of type E. + */ + public static class Key<E> { + private final Order order; + + private Key(Order order) { + this.order = Preconditions.checkNotNull(order); + } + } + + public static final Key<Artifact> LIBRARY = new Key<>(LINK_ORDER); + public static final Key<Artifact> IMPORTED_LIBRARY = new Key<>(LINK_ORDER); + + /** + * Single-architecture linked binaries to be combined for the final multi-architecture binary. + */ + public static final Key<Artifact> LINKED_BINARY = new Key<>(STABLE_ORDER); + + /** + * Indicates which libraries to load with {@code -force_load}. This is a subset of the union of + * the {@link #LIBRARY} and {@link #IMPORTED_LIBRARY} sets. + */ + public static final Key<Artifact> FORCE_LOAD_LIBRARY = new Key<>(LINK_ORDER); + + /** + * Libraries to pass with -force_load flags when setting the linkopts in Xcodegen. This is needed + * in addition to {@link #FORCE_LOAD_LIBRARY} because that one, contains a mixture of import + * archives (which are not built by Xcode) and built-from-source library archives (which are built + * by Xcode). Archives that are built by Xcode are placed directly under + * {@code BUILT_PRODUCTS_DIR} while those not built by Xcode appear somewhere in the Bazel + * workspace under {@code WORKSPACE_ROOT}. + */ + public static final Key<String> FORCE_LOAD_FOR_XCODEGEN = new Key<>(LINK_ORDER); + + public static final Key<Artifact> HEADER = new Key<>(STABLE_ORDER); + + /** + * Include search paths specified with {@code -I} on the command line. Also known as header search + * paths (and distinct from <em>user</em> header search paths). + */ + public static final Key<PathFragment> INCLUDE = new Key<>(LINK_ORDER); + + /** + * Key for values in {@code defines} attributes. These are passed as {@code -D} flags to all + * invocations of the compiler for this target and all depending targets. + */ + public static final Key<String> DEFINE = new Key<>(STABLE_ORDER); + + public static final Key<Artifact> ASSET_CATALOG = new Key<>(STABLE_ORDER); + + /** + * Added to {@link TargetControl#getGeneralResourceFileList()} when running Xcodegen. + */ + public static final Key<Artifact> GENERAL_RESOURCE_FILE = new Key<>(STABLE_ORDER); + + /** + * Exec paths of {@code .bundle} directories corresponding to imported bundles to link. + * These are passed to Xcodegen. + */ + public static final Key<PathFragment> BUNDLE_IMPORT_DIR = new Key<>(STABLE_ORDER); + + /** + * Files that are plopped into the final bundle at some arbitrary bundle path. Note that these are + * not passed to Xcodegen, and these don't include information about where the file originated + * from. + */ + public static final Key<BundleableFile> BUNDLE_FILE = new Key<>(STABLE_ORDER); + + public static final Key<PathFragment> XCASSETS_DIR = new Key<>(STABLE_ORDER); + public static final Key<String> SDK_DYLIB = new Key<>(STABLE_ORDER); + public static final Key<SdkFramework> SDK_FRAMEWORK = new Key<>(STABLE_ORDER); + public static final Key<SdkFramework> WEAK_SDK_FRAMEWORK = new Key<>(STABLE_ORDER); + public static final Key<Xcdatamodel> XCDATAMODEL = new Key<>(STABLE_ORDER); + public static final Key<Flag> FLAG = new Key<>(STABLE_ORDER); + + /** + * Merge zips to include in the bundle. The entries of these zip files are included in the final + * bundle with the same path. The entries in the merge zips should not include the bundle root + * path (e.g. {@code Foo.app}). + */ + public static final Key<Artifact> MERGE_ZIP = new Key<>(STABLE_ORDER); + + /** + * Exec paths of {@code .framework} directories corresponding to frameworks to link. These cause + * -F arguments (framework search paths) to be added to each compile action, and -framework (link + * framework) arguments to be added to each link action. + */ + public static final Key<PathFragment> FRAMEWORK_DIR = new Key<>(LINK_ORDER); + + /** + * Files in {@code .framework} directories that should be included as inputs when compiling and + * linking. + */ + public static final Key<Artifact> FRAMEWORK_FILE = new Key<>(STABLE_ORDER); + + /** + * Bundles which should be linked in as a nested bundle to the final application. + */ + public static final Key<Bundling> NESTED_BUNDLE = new Key<>(STABLE_ORDER); + + /** + * Artifact containing information on debug symbols + */ + public static final Key<Artifact> DEBUG_SYMBOLS = new Key<>(STABLE_ORDER); + + /** + * Flags that apply to a transitive build dependency tree. Each item in the enum corresponds to a + * flag. If the item is included in the key {@link #FLAG}, then the flag is considered set. + */ + public enum Flag { + /** + * Indicates that C++ (or Objective-C++) is used in any source file. This affects how the linker + * is invoked. + */ + USES_CPP; + } + + private final ImmutableMap<Key<?>, NestedSet<?>> items; + + // Items which should be passed to direct dependers, but not transitive dependers. + private final ImmutableMap<Key<?>, NestedSet<?>> nonPropagatedItems; + + private ObjcProvider( + ImmutableMap<Key<?>, NestedSet<?>> items, + ImmutableMap<Key<?>, NestedSet<?>> nonPropagatedItems) { + this.items = Preconditions.checkNotNull(items); + this.nonPropagatedItems = Preconditions.checkNotNull(nonPropagatedItems); + } + + /** + * All artifacts, bundleable files, etc. of the type specified by {@code key}. + */ + @SuppressWarnings("unchecked") + public <E> NestedSet<E> get(Key<E> key) { + Preconditions.checkNotNull(key); + NestedSetBuilder<E> builder = new NestedSetBuilder<>(key.order); + if (nonPropagatedItems.containsKey(key)) { + builder.addTransitive((NestedSet<E>) nonPropagatedItems.get(key)); + } + if (items.containsKey(key)) { + builder.addTransitive((NestedSet<E>) items.get(key)); + } + return builder.build(); + } + + /** + * Indicates whether {@code flag} is set on this provider. + */ + public boolean is(Flag flag) { + return Iterables.contains(get(FLAG), flag); + } + + /** + * Indicates whether this provider has any asset catalogs. This is true whenever some target in + * its transitive dependency tree specifies a non-empty {@code asset_catalogs} attribute. + */ + public boolean hasAssetCatalogs() { + return !get(XCASSETS_DIR).isEmpty(); + } + + /** + * A builder for this context with an API that is optimized for collecting information from + * several transitive dependencies. + */ + public static final class Builder { + private final Map<Key<?>, NestedSetBuilder<?>> items = new HashMap<>(); + private final Map<Key<?>, NestedSetBuilder<?>> nonPropagatedItems = new HashMap<>(); + + private static void maybeAddEmptyBuilder(Map<Key<?>, NestedSetBuilder<?>> set, Key<?> key) { + if (!set.containsKey(key)) { + set.put(key, new NestedSetBuilder<>(key.order)); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private void uncheckedAddAll(Key key, Iterable toAdd) { + maybeAddEmptyBuilder(items, key); + items.get(key).addAll(toAdd); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private void uncheckedAddTransitive(Key key, NestedSet toAdd, boolean propagate) { + Map<Key<?>, NestedSetBuilder<?>> set = propagate ? items : nonPropagatedItems; + maybeAddEmptyBuilder(set, key); + set.get(key).addTransitive(toAdd); + } + + /** + * Adds elements in items, and propagate them to any (transitive) dependers on this + * ObjcProvider. + */ + public <E> Builder addTransitiveAndPropagate(Key<E> key, NestedSet<E> items) { + uncheckedAddTransitive(key, items, true); + return this; + } + + /** + * Add all elements from provider, and propagate them to any (transitive) dependers on this + * ObjcProvider. + */ + public Builder addTransitiveAndPropagate(ObjcProvider provider) { + for (Map.Entry<Key<?>, NestedSet<?>> typeEntry : provider.items.entrySet()) { + uncheckedAddTransitive(typeEntry.getKey(), typeEntry.getValue(), true); + } + return this; + } + + /** + * Add all elements from a single key of the given provider, and propagate them to any + * (transitive) dependers on this ObjcProvider. + */ + public <E> Builder addTransitiveAndPropagate(Key<E> key, ObjcProvider provider) { + addTransitiveAndPropagate(key, provider.get(key)); + return this; + } + + /** + * Add all elements from providers, and propagate them to any (transitive) dependers on this + * ObjcProvider. + */ + public Builder addTransitiveAndPropagate(Iterable<ObjcProvider> providers) { + for (ObjcProvider provider : providers) { + addTransitiveAndPropagate(provider); + } + return this; + } + + /** + * Add elements from providers, but don't propagate them to any dependers on this ObjcProvider. + * These elements will be exposed to {@link #get(Key)} calls, but not to any ObjcProviders + * which add this provider to themself. + */ + public Builder addTransitiveWithoutPropagating(Iterable<ObjcProvider> providers) { + for (ObjcProvider provider : providers) { + for (Map.Entry<Key<?>, NestedSet<?>> typeEntry : provider.items.entrySet()) { + uncheckedAddTransitive(typeEntry.getKey(), typeEntry.getValue(), false); + } + } + return this; + } + + /** + * Add element, and propagate it to any (transitive) dependers on this ObjcProvider. + */ + public <E> Builder add(Key<E> key, E toAdd) { + uncheckedAddAll(key, ImmutableList.of(toAdd)); + return this; + } + + /** + * Add elements in toAdd, and propagate them to any (transitive) dependers on this ObjcProvider. + */ + public <E> Builder addAll(Key<E> key, Iterable<? extends E> toAdd) { + uncheckedAddAll(key, toAdd); + return this; + } + + public ObjcProvider build() { + ImmutableMap.Builder<Key<?>, NestedSet<?>> propagated = new ImmutableMap.Builder<>(); + for (Map.Entry<Key<?>, NestedSetBuilder<?>> typeEntry : items.entrySet()) { + propagated.put(typeEntry.getKey(), typeEntry.getValue().build()); + } + ImmutableMap.Builder<Key<?>, NestedSet<?>> nonPropagated = new ImmutableMap.Builder<>(); + for (Map.Entry<Key<?>, NestedSetBuilder<?>> typeEntry : nonPropagatedItems.entrySet()) { + nonPropagated.put(typeEntry.getKey(), typeEntry.getValue().build()); + } + return new ObjcProvider(propagated.build(), nonPropagated.build()); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java new file mode 100644 index 0000000000..ad1e4dc9c5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java @@ -0,0 +1,531 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BaseRuleClasses.BaseRule; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Shared utility code for Objective-C rules. + */ +public class ObjcRuleClasses { + + private ObjcRuleClasses() { + throw new UnsupportedOperationException("static-only"); + } + + /** + * Returns a derived Artifact by appending a String to a root-relative path. This is similar to + * {@link RuleContext#getRelatedArtifact(PathFragment, String)}, except the existing extension is + * not removed. + */ + static Artifact artifactByAppendingToRootRelativePath( + RuleContext ruleContext, PathFragment path, String suffix) { + return ruleContext.getAnalysisEnvironment().getDerivedArtifact( + path.replaceName(path.getBaseName() + suffix), + ruleContext.getBinOrGenfilesDirectory()); + } + + static IntermediateArtifacts intermediateArtifacts(RuleContext ruleContext) { + return new IntermediateArtifacts( + ruleContext.getAnalysisEnvironment(), ruleContext.getBinOrGenfilesDirectory(), + ruleContext.getLabel(), /*archiveFileNameSuffix=*/""); + } + + /** + * Returns a {@link IntermediateArtifacts} to be used to compile and link the ObjC source files + * in {@code j2ObjcSource}. + */ + static IntermediateArtifacts j2objcIntermediateArtifacts(RuleContext ruleContext, + J2ObjcSource j2ObjcSource) { + // We need to append "_j2objc" to the name of the generated archive file to distinguish it from + // the C/C++ archive file created by proto_library targets with attribute cc_api_version + // specified. + return new IntermediateArtifacts( + ruleContext.getAnalysisEnvironment(), + ruleContext.getConfiguration().getBinDirectory(), + j2ObjcSource.getTargetLabel(), + /*archiveFileNameSuffix=*/"_j2objc"); + } + + /** + * Returns a {@link J2ObjcSrcsProvider} with J2ObjC-generated ObjC file information from the + * current rule, and from rules that can be reached transitively through the "deps" attribute. + * + * @param ruleContext the rule context of the current rule + * @param currentSource J2ObjC-generated ObjC file information from the current rule to contribute + * to the returned provider + * @return a {@link J2ObjcSrcsProvider} containing {@code currentSources} and source information + * from the transitive closure. + */ + public static J2ObjcSrcsProvider j2ObjcSrcsProvider(RuleContext ruleContext, + J2ObjcSource currentSource) { + return j2ObjcSrcsProvider(ruleContext, Optional.of(currentSource)); + } + + /** + * Returns a {@link J2ObjcSrcsProvider} with J2ObjC-generated ObjC file information from rules + * that can be reached transitively through the "deps" attribute. + * + * @param ruleContext the rule context of the current rule + * @return a {@link J2ObjcSrcsProvider} containing source information from the transitive closure. + */ + public static J2ObjcSrcsProvider j2ObjcSrcsProvider(RuleContext ruleContext) { + return j2ObjcSrcsProvider(ruleContext, Optional.<J2ObjcSource>absent()); + } + + private static J2ObjcSrcsProvider j2ObjcSrcsProvider(RuleContext ruleContext, + Optional<J2ObjcSource> currentSource) { + NestedSetBuilder<J2ObjcSource> builder = NestedSetBuilder.stableOrder(); + builder.addAll(currentSource.asSet()); + boolean hasProtos = currentSource.isPresent() + && currentSource.get().getSourceType() == J2ObjcSource.SourceType.PROTO; + + for (J2ObjcSrcsProvider provider : + ruleContext.getPrerequisites("deps", Mode.TARGET, J2ObjcSrcsProvider.class)) { + builder.addTransitive(provider.getSrcs()); + hasProtos |= provider.hasProtos(); + } + + return new J2ObjcSrcsProvider(builder.build(), hasProtos); + } + + public static Artifact artifactByAppendingToBaseName(RuleContext context, String suffix) { + return artifactByAppendingToRootRelativePath( + context, context.getLabel().toPathFragment(), suffix); + } + + static ObjcActionsBuilder actionsBuilder(RuleContext ruleContext) { + return new ObjcActionsBuilder( + ruleContext, + intermediateArtifacts(ruleContext), + ObjcRuleClasses.objcConfiguration(ruleContext), + ruleContext.getConfiguration(), + ruleContext); + } + + public static ObjcConfiguration objcConfiguration(RuleContext ruleContext) { + return ruleContext.getFragment(ObjcConfiguration.class); + } + + @VisibleForTesting + static final Iterable<SdkFramework> AUTOMATIC_SDK_FRAMEWORKS = ImmutableList.of( + new SdkFramework("Foundation"), new SdkFramework("UIKit")); + + /** + * Attributes for {@code objc_*} rules that have compiler (and in the future, possibly linker) + * options + */ + @BlazeRule(name = "$objc_opts_rule", + type = RuleClassType.ABSTRACT) + public static class ObjcOptsRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE($objc_opts_rule).ATTRIBUTE(copts) --> + Extra flags to pass to the compiler. + ${SYNOPSIS} + Subject to <a href="#make_variables">"Make variable"</a> substitution and + <a href="#sh-tokenization">Bourne shell tokenization</a>. + These flags will only apply to this target, and not those upon which + it depends, or those which depend on it. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("copts", STRING_LIST)) + .build(); + } + } + + /** + * Attributes for {@code objc_*} rules that can link in SDK frameworks. + */ + @BlazeRule(name = "$objc_sdk_frameworks_rule", + type = RuleClassType.ABSTRACT) + public static class ObjcSdkFrameworksRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE($objc_sdk_frameworks_rule).ATTRIBUTE(sdk_frameworks) --> + Names of SDK frameworks to link with. For instance, "XCTest" or + "Cocoa". "UIKit" and "Foundation" are always included and do not mean + anything if you include them. + When linking a library, only those frameworks named in that library's + sdk_frameworks attribute are linked in. When linking a binary, all + SDK frameworks named in that binary's transitive dependency graph are + used. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("sdk_frameworks", STRING_LIST)) + /* <!-- #BLAZE_RULE($objc_sdk_frameworks_rule).ATTRIBUTE(weak_sdk_frameworks) --> + Names of SDK frameworks to weakly link with. For instance, + "MediaAccessibility". In difference to regularly linked SDK + frameworks, symbols from weakly linked frameworks do not cause an + error if they are not present at runtime. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("weak_sdk_frameworks", STRING_LIST)) + /* <!-- #BLAZE_RULE($objc_sdk_frameworks_rule).ATTRIBUTE(sdk_dylibs) --> + Names of SDK .dylib libraries to link with. For instance, "libz" or + "libarchive". "libc++" is included automatically if the binary has + any C++ or Objective-C++ sources in its dependency tree. When linking + a binary, all libraries named in that binary's transitive dependency + graph are used. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("sdk_dylibs", STRING_LIST)) + .build(); + } + } + + /** + * Iff a file matches this type, it is considered to use C++. + */ + static final FileType CPP_SOURCES = FileType.of(".cc", ".cpp", ".mm", ".cxx", ".C"); + + private static final FileType NON_CPP_SOURCES = FileType.of(".m", ".c"); + + static final FileTypeSet SRCS_TYPE = FileTypeSet.of(NON_CPP_SOURCES, CPP_SOURCES); + + static final FileTypeSet NON_ARC_SRCS_TYPE = FileTypeSet.of(FileType.of(".m", ".mm")); + + static final FileTypeSet PLIST_TYPE = FileTypeSet.of(FileType.of(".plist")); + + static final FileTypeSet STORYBOARD_TYPE = FileTypeSet.of(FileType.of(".storyboard")); + + static final FileType XIB_TYPE = FileType.of(".xib"); + + /** + * Common attributes for {@code objc_*} rules that allow the definition of resources such as + * storyboards. + */ + @BlazeRule(name = "$objc_base_resources_rule", + type = RuleClassType.ABSTRACT, + ancestors = { BaseRule.class }) + public static class ObjcBaseResourcesRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(strings) --> + Files which are plists of strings, often localizable. These files + are converted to binary plists (if they are not already) and placed + in the bundle root of the final package. If this file's immediate + containing directory is named *.lproj (e.g. en.lproj, Base.lproj), it + will be placed under a directory of that name in the final bundle. + This allows for localizable strings. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("strings", LABEL_LIST).legacyAllowAnyFileType() + .direct_compile_time_input()) + /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(xibs) --> + Files which are .xib resources, possibly localizable. These files are + compiled to .nib files and placed the bundle root of the final + package. If this file's immediate containing directory is named + *.lproj (e.g. en.lproj, Base.lproj), it will be placed under a + directory of that name in the final bundle. This allows for + localizable UI. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("xibs", LABEL_LIST) + .direct_compile_time_input() + .allowedFileTypes(XIB_TYPE)) + /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(storyboards) --> + Files which are .storyboard resources, possibly localizable. These + files are compiled to .storyboardc directories, which are placed in + the bundle root of the final package. If the storyboards's immediate + containing directory is named *.lproj (e.g. en.lproj, Base.lproj), it + will be placed under a directory of that name in the final bundle. + This allows for localizable UI. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("storyboards", LABEL_LIST) + .allowedFileTypes(STORYBOARD_TYPE)) + /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(resources) --> + Files to include in the final application bundle. They are not + processed or compiled in any way besides the processing done by the + rules that actually generate them. These files are placed in the root + of the bundle (e.g. Payload/foo.app/...) in most cases. However, if + they appear to be localized (i.e. are contained in a directory called + *.lproj), they will be placed in a directory of the same name in the + app bundle. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("resources", LABEL_LIST).legacyAllowAnyFileType().direct_compile_time_input()) + /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(datamodels) --> + Files that comprise the data models of the final linked binary. + Each file must have a containing directory named *.xcdatamodel, which + is usually contained by another *.xcdatamodeld (note the added d) + directory. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("datamodels", LABEL_LIST).legacyAllowAnyFileType() + .direct_compile_time_input()) + /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(asset_catalogs) --> + Files that comprise the asset catalogs of the final linked binary. + Each file must have a containing directory named *.xcassets. This + containing directory becomes the root of one of the asset catalogs + linked with any binary that depends directly or indirectly on this + target. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("asset_catalogs", LABEL_LIST).legacyAllowAnyFileType() + .direct_compile_time_input()) + .add(attr("$xcodegen", LABEL).cfg(HOST).exec() + .value(env.getLabel("//tools/objc:xcodegen"))) + .add(attr("$plmerge", LABEL).cfg(HOST).exec() + .value(env.getLabel("//tools/objc:plmerge"))) + .add(attr("$momczip_deploy", LABEL).cfg(HOST) + .value(env.getLabel("//tools/objc:momczip_deploy.jar"))) + .add(attr("$actooloribtoolzip_deploy", LABEL).cfg(HOST) + .value(env.getLabel("//tools/objc:actooloribtoolzip_deploy.jar"))) + .build(); + } + } + + /** + * Common attributes for {@code objc_*} rules that contain compilable content. + */ + @BlazeRule(name = "$objc_compilation_rule", + type = RuleClassType.ABSTRACT, + ancestors = { BaseRuleClasses.RuleBase.class, ObjcSdkFrameworksRule.class, + ObjcBaseResourcesRule.class }) + public static class ObjcCompilationRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE($objc_compilation_rule).ATTRIBUTE(hdrs) --> + The list of Objective-C files that are included as headers by source + files in this rule or by users of this library. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("hdrs", LABEL_LIST) + .direct_compile_time_input() + .allowedFileTypes(FileTypeSet.ANY_FILE)) + /* <!-- #BLAZE_RULE($objc_compilation_rule).ATTRIBUTE(includes) --> + List of <code>#include/#import</code> search paths to add to this target + and all depending targets. This is to support third party and + open-sourced libraries that do not specify the entire workspace path in + their <code>#import/#include</code> statements. + <p> + The paths are interpreted relative to the package directory, and the + genfiles and bin roots (e.g. <code>blaze-genfiles/pkg/includedir</code> + and <code>blaze-out/pkg/includedir</code>) are included in addition to the + actual client root. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("includes", Type.STRING_LIST)) + /* <!-- #BLAZE_RULE($objc_compilation_rule).ATTRIBUTE(sdk_includes) --> + List of <code>#include/#import</code> search paths to add to this target + and all depending targets, where each path is relative to + <code>$(SDKROOT)/usr/include</code>. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("sdk_includes", Type.STRING_LIST)) + .build(); + } + } + + /** + * Common attributes for rules that uses ObjC proto compiler. + */ + @BlazeRule(name = "$objc_proto_rule", + type = RuleClassType.ABSTRACT) + public static class ObjcProtoRule implements RuleDefinition { + + /** + * A Predicate that returns true if the ObjC proto compiler and its support deps are needed by + * the current rule. + * + * <p>For proto_library rules, this will return true if they have a j2objc_api_version + * attribute, and it is greater than 0. For other rules, this will return true by default. + */ + public static final Predicate<AttributeMap> USE_PROTO_COMPILER = new Predicate<AttributeMap>() { + @Override + public boolean apply(AttributeMap rule) { + return rule.getAttributeDefinition("j2objc_api_version") == null + || rule.get("j2objc_api_version", Type.INTEGER) != 0; + } + }; + + public static final String COMPILE_PROTOS_ATTR = "$googlemac_proto_compiler"; + public static final String PROTO_SUPPORT_ATTR = "$googlemac_proto_compiler_support"; + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .add(attr(COMPILE_PROTOS_ATTR, LABEL) + .allowedFileTypes(FileType.of(".py")) + .cfg(HOST) + .singleArtifact() + .condition(USE_PROTO_COMPILER) + .value(env.getLabel("//tools/objc:compile_protos"))) + .add(attr(PROTO_SUPPORT_ATTR, LABEL) + .legacyAllowAnyFileType() + .cfg(HOST) + .condition(USE_PROTO_COMPILER) + .value(env.getLabel("//tools/objc:proto_support"))) + .build(); + } + } + + /** + * Base rule definition for iOS test rules. + */ + @BlazeRule(name = "$ios_test_base_rule", + type = RuleClassType.ABSTRACT, + ancestors = { ObjcBinaryRule.class }) + public static class IosTestBaseRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE($ios_test_base_rule).ATTRIBUTE(target_device) --> + The device against which to run the test. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr(IosTest.TARGET_DEVICE, LABEL) + .allowedFileTypes() + .allowedRuleClasses("ios_device")) + /* <!-- #BLAZE_RULE($ios_test_base_rule).ATTRIBUTE(xctest) --> + Whether this target contains tests using the XCTest testing framework. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr(IosTest.IS_XCTEST, BOOLEAN)) + /* <!-- #BLAZE_RULE($ios_test_base_rule).ATTRIBUTE(xctest_app) --> + A <code>objc_binary</code> target that contains the app bundle to test against in XCTest. + This attribute is only valid if <code>xctest</code> is true. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr(IosTest.XCTEST_APP, LABEL) + .value(new Attribute.ComputedDefault(IosTest.IS_XCTEST) { + @Override + public Object getDefault(AttributeMap rule) { + return rule.get(IosTest.IS_XCTEST, Type.BOOLEAN) + ? env.getLabel("//tools/objc:xctest_app") + : null; + } + }) + .allowedFileTypes() + .allowedRuleClasses("objc_binary")) + .override(attr("infoplist", LABEL) + .value(new Attribute.ComputedDefault(IosTest.IS_XCTEST) { + @Override + public Object getDefault(AttributeMap rule) { + return rule.get(IosTest.IS_XCTEST, Type.BOOLEAN) + ? env.getLabel("//tools/objc:xctest_infoplist") + : null; + } + }) + .allowedFileTypes(PLIST_TYPE)) + .build(); + } + } + + /** + * Abstract rule type with the {@code infoplist} attribute. + */ + @BlazeRule(name = "$objc_has_infoplist_rule", + type = RuleClassType.ABSTRACT) + public static class ObjcHasInfoplistRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE($objc_has_infoplist_rule).ATTRIBUTE(infoplist) --> + The infoplist file. This corresponds to <i>appname</i>-Info.plist in Xcode projects. + ${SYNOPSIS} + Blaze will perform variable substitution on the plist file for the following values: + <ul> + <li><code>${EXECUTABLE_NAME}</code>: The name of the executable generated and included + in the bundle by blaze, which can be used as the value for + <code>CFBundleExecutable</code> within the plist. + <li><code>${BUNDLE_NAME}</code>: This target's name and bundle suffix (.bundle or .app) + in the form<code><var>name</var></code>.<code>suffix</code>. + <li><code>${PRODUCT_NAME}</code>: This target's name. + </ul> + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("infoplist", LABEL) + .allowedFileTypes(PLIST_TYPE)) + .build(); + } + } + + /** + * Abstract rule type with the {@code entitlements} attribute. + */ + @BlazeRule(name = "$objc_has_entitlements_rule", + type = RuleClassType.ABSTRACT) + public static class ObjcHasEntitlementsRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE($objc_has_entitlements_rule).ATTRIBUTE(entitlements) --> + The entitlements file required for device builds of this application. See + <a href="https://developer.apple.com/library/mac/documentation/Miscellaneous/Reference/EntitlementKeyReference/Chapters/AboutEntitlements.html">the apple documentation</a> + for more information. If absent, the default entitlements from the + provisioning profile will be used. + <p> + The following variables are substituted as per + <a href="https://developer.apple.com/library/ios/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html">their definitions in Apple's documentation</a>: + $(AppIdentifierPrefix) and $(CFBundleIdentifier). + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("entitlements", LABEL).legacyAllowAnyFileType()) + .build(); + } + } + + /** + * Object that supplies tools used by all rules which have the helper tools common to most rule + * implementations. + */ + static final class Tools { + private final RuleContext ruleContext; + + Tools(RuleContext ruleContext) { + this.ruleContext = Preconditions.checkNotNull(ruleContext); + } + + Artifact actooloribtoolzipDeployJar() { + return ruleContext.getPrerequisiteArtifact("$actooloribtoolzip_deploy", Mode.HOST); + } + + Artifact momczipDeployJar() { + return ruleContext.getPrerequisiteArtifact("$momczip_deploy", Mode.HOST); + } + + FilesToRunProvider xcodegen() { + return ruleContext.getExecutablePrerequisite("$xcodegen", Mode.HOST); + } + + FilesToRunProvider plmerge() { + return ruleContext.getExecutablePrerequisite("$plmerge", Mode.HOST); + } + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcSdkFrameworks.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcSdkFrameworks.java new file mode 100644 index 0000000000..2ff3a7c8d8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcSdkFrameworks.java @@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcSdkFrameworksRule; + +/** + * Common logic for rules that inherit from {@link ObjcSdkFrameworksRule}. + */ +public class ObjcSdkFrameworks { + + /** + * Class that handles extraction and processing of attributes common to inheritors of {@link + * ObjcSdkFrameworksRule}. + */ + public static class Attributes { + + private final RuleContext ruleContext; + + public Attributes(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + /** + * Returns the SDK frameworks defined on the rule's {@code sdk_frameworks} attribute as well as + * base frameworks defined in {@link ObjcRuleClasses#AUTOMATIC_SDK_FRAMEWORKS}. + */ + ImmutableSet<SdkFramework> sdkFrameworks() { + ImmutableSet.Builder<SdkFramework> result = new ImmutableSet.Builder<>(); + result.addAll(ObjcRuleClasses.AUTOMATIC_SDK_FRAMEWORKS); + for (String explicit : ruleContext.attributes().get("sdk_frameworks", Type.STRING_LIST)) { + result.add(new SdkFramework(explicit)); + } + return result.build(); + } + + /** + * Returns all SDK frameworks defined on the rule's {@code weak_sdk_frameworks} attribute. + */ + ImmutableSet<SdkFramework> weakSdkFrameworks() { + ImmutableSet.Builder<SdkFramework> result = new ImmutableSet.Builder<>(); + for (String frameworkName : + ruleContext.attributes().get("weak_sdk_frameworks", Type.STRING_LIST)) { + result.add(new SdkFramework(frameworkName)); + } + return result.build(); + } + + /** + * Returns all SDK dylibs defined on the rule's {@code sdk_dylibs} attribute. + */ + ImmutableSet<String> sdkDylibs() { + return ImmutableSet.copyOf(ruleContext.attributes().get("sdk_dylibs", Type.STRING_LIST)); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeproj.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeproj.java new file mode 100644 index 0000000000..e167f8966a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeproj.java @@ -0,0 +1,47 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * Implementation for {@code objc_xcodeproj}. + */ +public class ObjcXcodeproj implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + XcodeProvider.Project project = XcodeProvider.Project.fromTopLevelTargets( + ruleContext.getPrerequisites("deps", Mode.TARGET, XcodeProvider.class)); + Artifact pbxproj = ruleContext.getImplicitOutputArtifact(XcodeSupport.PBXPROJ); + + ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext); + actionsBuilder.registerXcodegenActions( + new ObjcRuleClasses.Tools(ruleContext), pbxproj, project); + + return new RuleConfiguredTargetBuilder(ruleContext) + .setFilesToBuild(NestedSetBuilder.create(Order.STABLE_ORDER, pbxproj)) + .addProvider(RunfilesProvider.class, RunfilesProvider.EMPTY) + .build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeprojRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeprojRule.java new file mode 100644 index 0000000000..0a6c4baf97 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeprojRule.java @@ -0,0 +1,78 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for {@code objc_xcodeproj}. + */ +@BlazeRule(name = "objc_xcodeproj", + factoryClass = ObjcXcodeproj.class, + ancestors = { BaseRuleClasses.RuleBase.class }) +public class ObjcXcodeprojRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /*<!-- #BLAZE_RULE(objc_xcodeproj).IMPLICIT_OUTPUTS --> + <ul> + <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: A combined Xcode project file + containing all the included targets which can be used to develop or build on a Mac.</li> + </ul> + <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/ + .setImplicitOutputsFunction(XcodeSupport.PBXPROJ) + /* <!-- #BLAZE_RULE(objc_xcodeproj).ATTRIBUTE(deps) --> + The list of targets to include in the combined Xcode project file. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .override(builder.copy("deps") + .nonEmpty() + .allowedRuleClasses( + "objc_binary", + "ios_test", + "objc_bundle_library", + "objc_import", + "objc_library")) + .override(attr("testonly", BOOLEAN) + .nonconfigurable("Must support test deps.") + .value(true)) + .add(attr("$xcodegen", LABEL) + .cfg(HOST) + .exec() + .value(env.getLabel("//tools/objc:xcodegen"))) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_xcodeproj, TYPE = OTHER, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule combines build information about several objc targets (and all their transitive +dependencies) into a single Xcode project file, for use in developing on a Mac.</p> + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/ diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/OptionsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/OptionsProvider.java new file mode 100644 index 0000000000..f87c96a66e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/OptionsProvider.java @@ -0,0 +1,87 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.xcode.util.Value; + +/** + * Provides information contained in a {@code objc_options} target. + */ +@Immutable +final class OptionsProvider + extends Value<OptionsProvider> + implements TransitiveInfoProvider { + static final class Builder { + private Iterable<String> copts = ImmutableList.of(); + private final NestedSetBuilder<Artifact> infoplists = NestedSetBuilder.stableOrder(); + + /** + * Adds copts to the end of the copts sequence. + */ + public Builder addCopts(Iterable<String> copts) { + this.copts = Iterables.concat(this.copts, copts); + return this; + } + + public Builder addInfoplists(Iterable<Artifact> infoplists) { + this.infoplists.addAll(infoplists); + return this; + } + + /** + * Adds infoplists and copts from the given provider, if present. copts are added to the end of + * the sequence. + */ + public Builder addTransitive(Optional<OptionsProvider> maybeProvider) { + for (OptionsProvider provider : maybeProvider.asSet()) { + this.copts = Iterables.concat(this.copts, provider.copts); + this.infoplists.addTransitive(provider.infoplists); + } + return this; + } + + public OptionsProvider build() { + return new OptionsProvider(ImmutableList.copyOf(copts), infoplists.build()); + } + } + + public static final OptionsProvider DEFAULT = new Builder().build(); + + private final ImmutableList<String> copts; + private final NestedSet<Artifact> infoplists; + + private OptionsProvider(ImmutableList<String> copts, NestedSet<Artifact> infoplists) { + super(copts, infoplists); + this.copts = Preconditions.checkNotNull(copts); + this.infoplists = Preconditions.checkNotNull(infoplists); + } + + public ImmutableList<String> getCopts() { + return copts; + } + + public NestedSet<Artifact> getInfoplists() { + return infoplists; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ResourceSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ResourceSupport.java new file mode 100644 index 0000000000..d1a717c165 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ResourceSupport.java @@ -0,0 +1,123 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; + +/** + * Support for resource processing on Objc rules. + * + * <p>Methods on this class can be called in any order without impacting the result. + */ +final class ResourceSupport { + private final RuleContext ruleContext; + private final Attributes attributes; + private final IntermediateArtifacts intermediateArtifacts; + private final Iterable<Xcdatamodel> datamodels; + + /** + * Creates a new resource support for the given context. + */ + ResourceSupport(RuleContext ruleContext) { + this.ruleContext = ruleContext; + this.attributes = new Attributes(ruleContext); + this.intermediateArtifacts = ObjcRuleClasses.intermediateArtifacts(ruleContext); + this.datamodels = Xcdatamodels.xcdatamodels(intermediateArtifacts, attributes.datamodels()); + } + + /** + * Registers resource generating actions (strings, storyboards, ...). + * + * @param storyboards storyboards defined by this rule + * + * @return this resource support + */ + ResourceSupport registerActions(Storyboards storyboards) { + ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext); + + ObjcRuleClasses.Tools tools = new ObjcRuleClasses.Tools(ruleContext); + actionsBuilder.registerResourceActions( + tools, + new ObjcActionsBuilder.StringsFiles( + CompiledResourceFile.fromStringsFiles(intermediateArtifacts, attributes.strings())), + new XibFiles(attributes.xibs()), + datamodels); + for (Artifact storyboardInput : storyboards.getInputs()) { + actionsBuilder.registerIbtoolzipAction( + tools, storyboardInput, intermediateArtifacts.compiledStoryboardZip(storyboardInput)); + } + return this; + } + + /** + * Adds common xcode settings to the given provider builder. + * + * @return this resource support + */ + ResourceSupport addXcodeSettings(XcodeProvider.Builder xcodeProviderBuilder) { + xcodeProviderBuilder.addInputsToXcodegen(Xcdatamodel.inputsToXcodegen(datamodels)); + return this; + } + + /** + * Validates resource attributes on this rule. + * + * @return this resource support + */ + ResourceSupport validateAttributes() { + Iterable<String> assetCatalogErrors = ObjcCommon.notInContainerErrors( + attributes.assetCatalogs(), ObjcCommon.ASSET_CATALOG_CONTAINER_TYPE); + for (String error : assetCatalogErrors) { + ruleContext.attributeError("asset_catalogs", error); + } + + Iterable<String> dataModelErrors = + ObjcCommon.notInContainerErrors(attributes.datamodels(), Xcdatamodels.CONTAINER_TYPES); + for (String error : dataModelErrors) { + ruleContext.attributeError("datamodels", error); + } + + return this; + } + + private static class Attributes { + private final RuleContext ruleContext; + + Attributes(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + ImmutableList<Artifact> datamodels() { + return ruleContext.getPrerequisiteArtifacts("datamodels", Mode.TARGET).list(); + } + + ImmutableList<Artifact> xibs() { + return ruleContext.getPrerequisiteArtifacts("xibs", Mode.TARGET) + .errorsForNonMatching(ObjcRuleClasses.XIB_TYPE) + .list(); + } + + ImmutableList<Artifact> strings() { + return ruleContext.getPrerequisiteArtifacts("strings", Mode.TARGET).list(); + } + + ImmutableList<Artifact> assetCatalogs() { + return ruleContext.getPrerequisiteArtifacts("asset_catalogs", Mode.TARGET).list(); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/SdkFramework.java b/src/main/java/com/google/devtools/build/lib/rules/objc/SdkFramework.java new file mode 100644 index 0000000000..c692fcd402 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/SdkFramework.java @@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.xcode.util.Value; + +/** + * Represents the name of an SDK framework. + * <p> + * Besides being a glorified String, this class prevents you from adding framework names to an + * argument list without explicitly specifying how to prefix them. + */ +final class SdkFramework extends Value<SdkFramework> { + private final String name; + + public SdkFramework(String name) { + super(name); + this.name = name; + } + + public String getName() { + return name; + } + + /** + * Returns an iterable which contains the name of each given framework in the same order. + */ + static Iterable<String> names(Iterable<SdkFramework> frameworks) { + ImmutableList.Builder<String> result = new ImmutableList.Builder<>(); + for (SdkFramework framework : frameworks) { + result.add(framework.getName()); + } + return result.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/Storyboards.java b/src/main/java/com/google/devtools/build/lib/rules/objc/Storyboards.java new file mode 100644 index 0000000000..204c22d598 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/Storyboards.java @@ -0,0 +1,76 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; + +/** + * Contains information about storyboards for a single target. This does not include information + * about the transitive closure. A storyboard: + * <ul> + * <li>Is a single file with an extension of {@code .storyboard} in its uncompiled, checked-in + * form. + * <li>Can be in a localized {@code .lproj} directory, including {@code Base.lproj}. + * <li>Compiles with {@code ibtool} to a directory with extension {@code .storyboardc} (note the + * added "c") + * </ul> + * + * <p>The {@link NestedSet}s stored in this class are only one level deep, and do not include the + * storyboards in the transitive closure. This is to facilitate structural sharing between copies + * of the sequences - the output zips can be added transitively to the inputs of the merge bundle + * action, as well as to the files to build set, and only one instance of the sequence exists for + * each set. + */ +final class Storyboards { + private final NestedSet<Artifact> outputZips; + private final NestedSet<Artifact> inputs; + + private Storyboards(NestedSet<Artifact> outputZips, NestedSet<Artifact> inputs) { + this.outputZips = outputZips; + this.inputs = inputs; + } + + public NestedSet<Artifact> getOutputZips() { + return outputZips; + } + + public NestedSet<Artifact> getInputs() { + return inputs; + } + + static Storyboards empty() { + return new Storyboards( + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER)); + } + + /** + * Generates a set of new instances given the raw storyboard inputs. + * @param inputs the {@code .storyboard} files. + * @param intermediateArtifacts the object used to determine the output zip {@link Artifact}s. + */ + static Storyboards fromInputs( + Iterable<Artifact> inputs, IntermediateArtifacts intermediateArtifacts) { + NestedSetBuilder<Artifact> outputZips = NestedSetBuilder.stableOrder(); + for (Artifact input : inputs) { + outputZips.add(intermediateArtifacts.compiledStoryboardZip(input)); + } + return new Storyboards(outputZips.build(), NestedSetBuilder.wrap(Order.STABLE_ORDER, inputs)); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XcTestAppProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XcTestAppProvider.java new file mode 100644 index 0000000000..1fc5d5dbce --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XcTestAppProvider.java @@ -0,0 +1,57 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Supplies information needed when a dependency serves as an {@code xctest_app}. + */ +@Immutable +final class XcTestAppProvider implements TransitiveInfoProvider { + private final Artifact bundleLoader; + private final Artifact ipa; + private final ObjcProvider objcProvider; + + XcTestAppProvider(Artifact bundleLoader, Artifact ipa, ObjcProvider objcProvider) { + this.bundleLoader = Preconditions.checkNotNull(bundleLoader); + this.ipa = Preconditions.checkNotNull(ipa); + this.objcProvider = Preconditions.checkNotNull(objcProvider); + } + + /** + * The bundle loader, which corresponds to the test app's binary. + */ + public Artifact getBundleLoader() { + return bundleLoader; + } + + public Artifact getIpa() { + return ipa; + } + + /** + * An {@link ObjcProvider} that should be included by any test target that uses this app as its + * {@code xctest_app}. This is <strong>not</strong> a typical {@link ObjcProvider} - it has + * certain linker-releated keys omitted, such as {@link ObjcProvider#LIBRARY}, since XcTests have + * access to symbols in their test rig without linking them into the main test binary. + */ + public ObjcProvider getObjcProvider() { + return objcProvider; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodel.java b/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodel.java new file mode 100644 index 0000000000..5b29435d36 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodel.java @@ -0,0 +1,136 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.util.Value; + +/** + * Represents an .xcdatamodel[d] directory - knowing all {@code Artifact}s contained therein - and + * the .zip file that it is compiled to which should be merged with the final application bundle. + * <p> + * An .xcdatamodel (here and below note that lack or presence of a d) directory contains the schema + * for a managed object, or a managed object model. It typically has two files: {@code layout} and + * {@code contents}, although this detail isn't addressed in Bazel code. Directories of this + * sort are compiled into a single .mom file. If the .xcdatamodel directory is inside a + * .xcdatamodeld directory, then the .mom file is placed inside a .momd directory. The .momd + * directory or .mom file is placed in the bundle root of the final bundle. + * <p> + * An .xcdatamodeld directory contains several .xcdatamodel directories, each corresponding to a + * different version. In addition the .xcdatamodeld directory contains a {@code .xccurrentversion} + * file which identifies the current version. (this file is also not handled explicitly by Bazel + * code). + * <p> + * When processing artifacts referenced by a {@code datamodels} attribute, we must determine if it + * is in a .xcdatamodeld directory or only a .xcdatamodel directory. We also must group the + * artifacts by their container, the container being an .xcdatamodeld directory if possible, and a + * .xcdatamodel directory otherwise. Every container is compiled with a single invocation of the + * Managed Object Model Compiler (momc) and corresponds to exactly one instance of this class. We + * invoke momc indirectly through the momczip tool (part of Bazel) which runs momc and zips the + * output. The files in this zip are placed in the bundle root of the final application, not unlike + * the zips generated by {@code actooloribtoolzip}. + */ +class Xcdatamodel extends Value<Xcdatamodel> { + private final Artifact outputZip; + private final ImmutableSet<Artifact> inputs; + private final PathFragment container; + + Xcdatamodel(Artifact outputZip, ImmutableSet<Artifact> inputs, PathFragment container) { + super(ImmutableMap.of( + "outputZip", outputZip, + "inputs", inputs, + "container", container)); + this.outputZip = outputZip; + this.inputs = inputs; + this.container = container; + } + + /** + * Returns the files that should be supplied to Xcodegen when generating a project that includes + * all of the given xcdatamodels. + */ + public static Iterable<Artifact> inputsToXcodegen(Iterable<Xcdatamodel> datamodels) { + ImmutableSet.Builder<Artifact> inputs = new ImmutableSet.Builder<>(); + for (Xcdatamodel datamodel : datamodels) { + for (Artifact generalInput : datamodel.inputs) { + if (generalInput.getExecPath().getBaseName().equals(".xccurrentversion")) { + inputs.add(generalInput); + } + } + } + return inputs.build(); + } + + public Artifact getOutputZip() { + return outputZip; + } + + /** + * Returns every known file in the container. This is every input file that is processed by momc. + */ + public ImmutableSet<Artifact> getInputs() { + return inputs; + } + + public PathFragment getContainer() { + return container; + } + + /** + * The ARCHIVE_ROOT passed to momczip. The archive root is the name of the .mom file + * unversioned object models, and the name of the .momd directory for versioned object models. + */ + public String archiveRootForMomczip() { + return name() + (container.getBaseName().endsWith(".xcdatamodeld") ? ".momd" : ".mom"); + } + + /** + * The name of the data model. This is the name of the container without the extension. For + * instance, if the container is "foo/Information.xcdatamodel" or "bar/Information.xcdatamodeld", + * then the name is "Information". + */ + public String name() { + String baseContainerName = container.getBaseName(); + int lastDot = baseContainerName.lastIndexOf('.'); + return baseContainerName.substring(0, lastDot); + } + + public static Iterable<Artifact> outputZips(Iterable<Xcdatamodel> models) { + return Iterables.transform(models, new Function<Xcdatamodel, Artifact>() { + @Override + public Artifact apply(Xcdatamodel model) { + return model.getOutputZip(); + } + }); + } + + /** + * Returns a sequence of all unique *.xcdatamodel directories that contain all the artifacts of + * the given models. Note that this does not return any *.xcdatamodeld directories. + */ + static Iterable<PathFragment> xcdatamodelDirs(Iterable<Xcdatamodel> models) { + ImmutableSet.Builder<PathFragment> result = new ImmutableSet.Builder<>(); + for (Xcdatamodel model : models) { + result.addAll(ObjcCommon.uniqueContainers(model.getInputs(), FileType.of(".xcdatamodel"))); + } + return result.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodels.java b/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodels.java new file mode 100644 index 0000000000..32d48aa696 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodels.java @@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.Map; + +/** + * Utility code for getting information specific to xcdatamodels for a single rule. + */ +class Xcdatamodels { + private Xcdatamodels() {} + + static final ImmutableList<FileType> CONTAINER_TYPES = + ImmutableList.of(FileType.of(".xcdatamodeld"), FileType.of(".xcdatamodel")); + + static Iterable<Xcdatamodel> xcdatamodels( + IntermediateArtifacts intermediateArtifacts, Iterable<Artifact> xcdatamodels) { + ImmutableSet.Builder<Xcdatamodel> result = new ImmutableSet.Builder<>(); + Multimap<PathFragment, Artifact> artifactsByContainer = byContainer(xcdatamodels); + + for (Map.Entry<PathFragment, Collection<Artifact>> modelDirEntry : + artifactsByContainer.asMap().entrySet()) { + PathFragment container = modelDirEntry.getKey(); + Artifact outputZip = intermediateArtifacts.compiledMomZipArtifact(container); + result.add( + new Xcdatamodel(outputZip, ImmutableSet.copyOf(modelDirEntry.getValue()), container)); + } + + return result.build(); + } + + + /** + * Arrange a sequence of artifacts into entries of a multimap by their nearest container + * directory, preferring {@code .xcdatamodeld} over {@code .xcdatamodel}. + * If an artifact is not inside any containing directory, then it is not present in the returned + * map. + */ + static Multimap<PathFragment, Artifact> byContainer(Iterable<Artifact> artifacts) { + ImmutableSetMultimap.Builder<PathFragment, Artifact> result = + new ImmutableSetMultimap.Builder<>(); + for (Artifact artifact : artifacts) { + for (PathFragment modelDir : + ObjcCommon.nearestContainerMatching(CONTAINER_TYPES, artifact).asSet()) { + result.put(modelDir, artifact); + } + } + return result.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProductType.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProductType.java new file mode 100644 index 0000000000..1a68206d61 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProductType.java @@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +/** + * Possible values that {@code objc_*} rules care about for what Xcode project files refer to as + * "product type." + */ +enum XcodeProductType { + LIBRARY_STATIC("com.apple.product-type.library.static"), + BUNDLE("com.apple.product-type.bundle"), + APPLICATION("com.apple.product-type.application"), + UNIT_TEST("com.apple.product-type.bundle.unit-test"), + EXTENSION("com.apple.product-type.app-extension"); + + private final String identifier; + + XcodeProductType(String identifier) { + this.identifier = identifier; + } + + /** + * Returns the string used to identify this product type in the {@code productType} field of + * {@code PBXNativeTarget} objects in Xcode project files. + */ + public String getIdentifier() { + return identifier; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProvider.java new file mode 100644 index 0000000000..b244cd9839 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProvider.java @@ -0,0 +1,452 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_IMPORT_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.DEFINE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FORCE_LOAD_FOR_XCODEGEN; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.GENERAL_RESOURCE_FILE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_DYLIB; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_FRAMEWORK; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.WEAK_SDK_FRAMEWORK; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCASSETS_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCDATAMODEL; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.objc.ObjcProvider.Flag; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.util.Interspersing; +import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.DependencyControl; +import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.TargetControl; +import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.XcodeprojBuildSetting; + +import java.util.EnumSet; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Provider which provides transitive dependency information that is specific to Xcodegen. In + * particular, it provides a sequence of targets which can be used to create a self-contained + * {@code .xcodeproj} file. + */ +@Immutable +public final class XcodeProvider implements TransitiveInfoProvider { + /** + * A builder for instances of {@link XcodeProvider}. + */ + public static final class Builder { + private Label label; + private final NestedSetBuilder<String> userHeaderSearchPaths = NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<String> headerSearchPaths = NestedSetBuilder.stableOrder(); + private Optional<InfoplistMerging> infoplistMerging = Optional.absent(); + private final NestedSetBuilder<XcodeProvider> dependencies = NestedSetBuilder.stableOrder(); + private final ImmutableList.Builder<XcodeprojBuildSetting> xcodeprojBuildSettings = + new ImmutableList.Builder<>(); + private final ImmutableList.Builder<String> copts = new ImmutableList.Builder<>(); + private final ImmutableList.Builder<String> compilationModeCopts = + new ImmutableList.Builder<>(); + private XcodeProductType productType; + private final ImmutableList.Builder<Artifact> headers = new ImmutableList.Builder<>(); + private Optional<CompilationArtifacts> compilationArtifacts = Optional.absent(); + private ObjcProvider objcProvider; + private Optional<XcodeProvider> testHost = Optional.absent(); + private final NestedSetBuilder<Artifact> inputsToXcodegen = NestedSetBuilder.stableOrder(); + + /** + * Sets the label of the build target which corresponds to this Xcode target. + */ + public Builder setLabel(Label label) { + this.label = label; + return this; + } + + /** + * Adds user header search paths for this target. + */ + public Builder addUserHeaderSearchPaths(Iterable<PathFragment> userHeaderSearchPaths) { + this.userHeaderSearchPaths.addAll(rootEach("$(WORKSPACE_ROOT)", userHeaderSearchPaths)); + return this; + } + + /** + * Adds header search paths for this target. Each path is interpreted relative to the given + * root, such as {@code "$(WORKSPACE_ROOT)"}. + */ + public Builder addHeaderSearchPaths(String root, Iterable<PathFragment> paths) { + this.headerSearchPaths.addAll(rootEach(root, paths)); + return this; + } + + /** + * Sets the Info.plist merging information. Used for applications. May be + * absent for other bundles. + */ + public Builder setInfoplistMerging(InfoplistMerging infoplistMerging) { + this.infoplistMerging = Optional.of(infoplistMerging); + return this; + } + + /** + * Adds items in the {@link NestedSet}s of the given target to the corresponding sets in this + * builder. This is useful if the given target is a dependency or like a dependency + * (e.g. a test host). The given provider is not registered as a dependency with this provider. + */ + private void addTransitiveSets(XcodeProvider dependencyish) { + inputsToXcodegen.addTransitive(dependencyish.inputsToXcodegen); + userHeaderSearchPaths.addTransitive(dependencyish.userHeaderSearchPaths); + headerSearchPaths.addTransitive(dependencyish.headerSearchPaths); + } + + /** + * Adds {@link XcodeProvider}s corresponding to direct dependencies of this target which should + * be added in the {@code .xcodeproj} file. + */ + public Builder addDependencies(Iterable<XcodeProvider> dependencies) { + for (XcodeProvider dependency : dependencies) { + this.dependencies.add(dependency); + this.dependencies.addTransitive(dependency.dependencies); + this.addTransitiveSets(dependency); + } + return this; + } + + /** + * Adds additional build settings of this target. + */ + public Builder addXcodeprojBuildSettings( + Iterable<XcodeprojBuildSetting> xcodeprojBuildSettings) { + this.xcodeprojBuildSettings.addAll(xcodeprojBuildSettings); + return this; + } + + /** + * Sets the copts to use when compiling the Xcode target. + */ + public Builder addCopts(Iterable<String> copts) { + this.copts.addAll(copts); + return this; + } + + /** + * Sets the copts derived from compilation mode to use when compiling the Xcode target. These + * will be included before the DEFINE options. + */ + public Builder addCompilationModeCopts(Iterable<String> copts) { + this.compilationModeCopts.addAll(copts); + return this; + } + + /** + * Sets the product type for the PBXTarget in the .xcodeproj file. + */ + public Builder setProductType(XcodeProductType productType) { + this.productType = productType; + return this; + } + + /** + * Adds to the header files of this target. It needs not to include the header files of + * dependencies. + */ + public Builder addHeaders(Iterable<Artifact> headers) { + this.headers.addAll(headers); + return this; + } + + /** + * The compilation artifacts for this target. + */ + public Builder setCompilationArtifacts(CompilationArtifacts compilationArtifacts) { + this.compilationArtifacts = Optional.of(compilationArtifacts); + return this; + } + + /** + * Sets the {@link ObjcProvider} corresponding to this target. + */ + public Builder setObjcProvider(ObjcProvider objcProvider) { + this.objcProvider = objcProvider; + return this; + } + + /** + * Sets the test host. This is used for xctest targets. + */ + public Builder setTestHost(XcodeProvider testHost) { + Preconditions.checkState(!this.testHost.isPresent()); + this.testHost = Optional.of(testHost); + this.addTransitiveSets(testHost); + return this; + } + + /** + * Adds inputs that are passed to Xcodegen when generating the project file. + */ + public Builder addInputsToXcodegen(Iterable<Artifact> inputsToXcodegen) { + this.inputsToXcodegen.addAll(inputsToXcodegen); + return this; + } + + public XcodeProvider build() { + Preconditions.checkArgument( + !testHost.isPresent() || (productType == XcodeProductType.UNIT_TEST), + "%s product types cannot have a test host (test host: %s).", productType, testHost); + return new XcodeProvider(this); + } + } + + /** + * A collection of top-level targets that can be used to create a complete project. + */ + public static final class Project { + private final NestedSet<Artifact> inputsToXcodegen; + private final ImmutableList<XcodeProvider> topLevelTargets; + + private Project( + NestedSet<Artifact> inputsToXcodegen, ImmutableList<XcodeProvider> topLevelTargets) { + this.inputsToXcodegen = inputsToXcodegen; + this.topLevelTargets = topLevelTargets; + } + + public static Project fromTopLevelTarget(XcodeProvider topLevelTarget) { + return fromTopLevelTargets(ImmutableList.of(topLevelTarget)); + } + + public static Project fromTopLevelTargets(Iterable<XcodeProvider> topLevelTargets) { + NestedSetBuilder<Artifact> inputsToXcodegen = NestedSetBuilder.stableOrder(); + for (XcodeProvider target : topLevelTargets) { + inputsToXcodegen.addTransitive(target.inputsToXcodegen); + } + return new Project(inputsToXcodegen.build(), ImmutableList.copyOf(topLevelTargets)); + } + + /** + * Returns artifacts that are passed to the Xcodegen action when generating a project file that + * contains all of the given targets. + */ + public NestedSet<Artifact> getInputsToXcodegen() { + return inputsToXcodegen; + } + + public ImmutableList<XcodeProvider> getTopLevelTargets() { + return topLevelTargets; + } + + /** + * Returns all the target controls that must be added to the xcodegen control. No other target + * controls are needed to generate a functional project file. This method creates a new list + * whenever it is called. + */ + public ImmutableList<TargetControl> targets() { + // Collect all the dependencies of all the providers, filtering out duplicates. + Set<XcodeProvider> providerSet = new LinkedHashSet<>(); + for (XcodeProvider target : topLevelTargets) { + Iterables.addAll(providerSet, target.providers()); + } + + ImmutableList.Builder<TargetControl> controls = new ImmutableList.Builder<>(); + for (XcodeProvider provider : providerSet) { + controls.add(provider.targetControl()); + } + return controls.build(); + } + } + + private final Label label; + private final NestedSet<String> userHeaderSearchPaths; + private final NestedSet<String> headerSearchPaths; + private final Optional<InfoplistMerging> infoplistMerging; + private final NestedSet<XcodeProvider> dependencies; + private final ImmutableList<XcodeprojBuildSetting> xcodeprojBuildSettings; + private final ImmutableList<String> copts; + private final ImmutableList<String> compilationModeCopts; + private final XcodeProductType productType; + private final ImmutableList<Artifact> headers; + private final Optional<CompilationArtifacts> compilationArtifacts; + private final ObjcProvider objcProvider; + private final Optional<XcodeProvider> testHost; + private final NestedSet<Artifact> inputsToXcodegen; + + private XcodeProvider(Builder builder) { + this.label = Preconditions.checkNotNull(builder.label); + this.userHeaderSearchPaths = builder.userHeaderSearchPaths.build(); + this.headerSearchPaths = builder.headerSearchPaths.build(); + this.infoplistMerging = builder.infoplistMerging; + this.dependencies = builder.dependencies.build(); + this.xcodeprojBuildSettings = builder.xcodeprojBuildSettings.build(); + this.copts = builder.copts.build(); + this.compilationModeCopts = builder.compilationModeCopts.build(); + this.productType = Preconditions.checkNotNull(builder.productType); + this.headers = builder.headers.build(); + this.compilationArtifacts = builder.compilationArtifacts; + this.objcProvider = Preconditions.checkNotNull(builder.objcProvider); + this.testHost = Preconditions.checkNotNull(builder.testHost); + this.inputsToXcodegen = builder.inputsToXcodegen.build(); + } + + /** + * Creates a builder whose values are all initialized to this provider. + */ + public Builder toBuilder() { + Builder builder = new Builder(); + builder.label = label; + builder.userHeaderSearchPaths.addAll(userHeaderSearchPaths); + builder.headerSearchPaths.addTransitive(headerSearchPaths); + builder.infoplistMerging = infoplistMerging; + builder.dependencies.addTransitive(dependencies); + builder.xcodeprojBuildSettings.addAll(xcodeprojBuildSettings); + builder.copts.addAll(copts); + builder.productType = productType; + builder.headers.addAll(headers); + builder.compilationArtifacts = compilationArtifacts; + builder.objcProvider = objcProvider; + builder.testHost = testHost; + builder.inputsToXcodegen.addTransitive(inputsToXcodegen); + return builder; + } + + /** + * Returns a list of this provider and all its transitive dependencies. + */ + private Iterable<XcodeProvider> providers() { + Set<XcodeProvider> providers = new LinkedHashSet<>(); + providers.add(this); + Iterables.addAll(providers, dependencies); + for (XcodeProvider justTestHost : testHost.asSet()) { + providers.add(justTestHost); + Iterables.addAll(providers, justTestHost.dependencies); + } + return ImmutableList.copyOf(providers); + } + + private static final EnumSet<XcodeProductType> CAN_LINK_PRODUCT_TYPES = EnumSet.of( + XcodeProductType.APPLICATION, XcodeProductType.BUNDLE, XcodeProductType.UNIT_TEST); + + private TargetControl targetControl() { + String buildFilePath = label.getPackageFragment().getSafePathString() + "/BUILD"; + // TODO(bazel-team): Add provisioning profile information when Xcodegen supports it. + TargetControl.Builder targetControl = TargetControl.newBuilder() + .setName(label.getName()) + .setLabel(label.toString()) + .setProductType(productType.getIdentifier()) + .addAllImportedLibrary(Artifact.toExecPaths(objcProvider.get(IMPORTED_LIBRARY))) + .addAllUserHeaderSearchPath(userHeaderSearchPaths) + .addAllHeaderSearchPath(headerSearchPaths) + .addAllSupportFile(Artifact.toExecPaths(headers)) + .addAllCopt(compilationModeCopts) + .addAllCopt(Interspersing.prependEach("-D", objcProvider.get(DEFINE))) + .addAllCopt(copts) + .addAllLinkopt( + Interspersing.beforeEach("-force_load", objcProvider.get(FORCE_LOAD_FOR_XCODEGEN))) + .addAllLinkopt(IosSdkCommands.DEFAULT_LINKER_FLAGS) + .addAllLinkopt(Interspersing.beforeEach( + "-weak_framework", SdkFramework.names(objcProvider.get(WEAK_SDK_FRAMEWORK)))) + .addAllBuildSetting(xcodeprojBuildSettings) + .addAllBuildSetting(IosSdkCommands.defaultWarningsForXcode()) + .addAllSdkFramework(SdkFramework.names(objcProvider.get(SDK_FRAMEWORK))) + .addAllFramework(PathFragment.safePathStrings(objcProvider.get(FRAMEWORK_DIR))) + .addAllXcassetsDir(PathFragment.safePathStrings(objcProvider.get(XCASSETS_DIR))) + .addAllXcdatamodel(PathFragment.safePathStrings( + Xcdatamodel.xcdatamodelDirs(objcProvider.get(XCDATAMODEL)))) + .addAllBundleImport(PathFragment.safePathStrings(objcProvider.get(BUNDLE_IMPORT_DIR))) + .addAllSdkDylib(objcProvider.get(SDK_DYLIB)) + .addAllGeneralResourceFile(Artifact.toExecPaths(objcProvider.get(GENERAL_RESOURCE_FILE))) + .addSupportFile(buildFilePath); + + if (CAN_LINK_PRODUCT_TYPES.contains(productType)) { + for (XcodeProvider dependency : dependencies) { + // Only add a library target to a binary's dependencies if it has source files to compile. + // Xcode cannot build targets without a source file in the PBXSourceFilesBuildPhase, so if + // such a target is present in the control file, it is only to get Xcodegen to put headers + // and resources not used by the final binary in the Project Navigator. + // + // The exception to this rule is the objc_bundle_library target. Bundles are generally used + // for resources and can lack a PBXSourceFilesBuildPhase in the project file and still be + // considered valid by Xcode. + boolean hasSources = dependency.compilationArtifacts.isPresent() + && dependency.compilationArtifacts.get().getArchive().isPresent(); + if (hasSources || (dependency.productType == XcodeProductType.BUNDLE)) { + targetControl.addDependency(DependencyControl.newBuilder() + .setTargetLabel(dependency.label.toString()) + .build()); + } + } + for (XcodeProvider justTestHost : testHost.asSet()) { + targetControl.addDependency(DependencyControl.newBuilder() + .setTargetLabel(justTestHost.label.toString()) + .setTestHost(true) + .build()); + } + } + + for (InfoplistMerging merging : infoplistMerging.asSet()) { + for (Artifact infoplist : merging.getPlistWithEverything().asSet()) { + targetControl.setInfoplist(infoplist.getExecPathString()); + } + } + for (CompilationArtifacts artifacts : compilationArtifacts.asSet()) { + targetControl + .addAllSourceFile(Artifact.toExecPaths(artifacts.getSrcs())) + .addAllNonArcSourceFile(Artifact.toExecPaths(artifacts.getNonArcSrcs())); + + for (Artifact pchFile : artifacts.getPchFile().asSet()) { + targetControl + .setPchPath(pchFile.getExecPathString()) + .addSupportFile(pchFile.getExecPathString()); + } + } + + if (objcProvider.is(Flag.USES_CPP)) { + targetControl.addSdkDylib("libc++"); + } + + return targetControl.build(); + } + + /** + * Prepends the given path to each path in {@code paths}. Empty paths are + * transformed to the value of {@code variable} rather than {@code variable + "/."} + */ + @VisibleForTesting + static Iterable<String> rootEach(final String prefix, Iterable<PathFragment> paths) { + Preconditions.checkArgument(prefix.startsWith("$"), + "prefix should start with a build setting variable like '$(NAME)': %s", prefix); + Preconditions.checkArgument(!prefix.endsWith("/"), + "prefix should not end with '/': %s", prefix); + return Iterables.transform(paths, new Function<PathFragment, String>() { + @Override + public String apply(PathFragment input) { + if (input.getSafePathString().equals(".")) { + return prefix; + } else { + return prefix + "/" + input.getSafePathString(); + } + } + }); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeSupport.java new file mode 100644 index 0000000000..f64c6bd6ea --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeSupport.java @@ -0,0 +1,102 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction; + +/** + * Support for Objc rule types that export an Xcode provider or generate xcode project files. + * + * <p>Methods on this class can be called in any order without impacting the result. + */ +public final class XcodeSupport { + + /** + * Template for a target's xcode project. + */ + public static final SafeImplicitOutputsFunction PBXPROJ = + fromTemplates("%{name}.xcodeproj/project.pbxproj"); + + private final RuleContext ruleContext; + + /** + * Creates a new xcode support for the given context. + */ + XcodeSupport(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + /** + * Adds xcode project files to the given builder. + * + * @return this xcode support + */ + XcodeSupport addFilesToBuild(NestedSetBuilder<Artifact> filesToBuild) { + filesToBuild.add(ruleContext.getImplicitOutputArtifact(PBXPROJ)); + return this; + } + + /** + * Registers actions that generate the rule's Xcode project. + * + * @param xcodeProvider information about this rule's xcode settings and that of its dependencies + * @return this xcode support + */ + XcodeSupport registerActions(XcodeProvider xcodeProvider) { + ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext); + actionsBuilder.registerXcodegenActions( + new ObjcRuleClasses.Tools(ruleContext), + ruleContext.getImplicitOutputArtifact(XcodeSupport.PBXPROJ), + XcodeProvider.Project.fromTopLevelTarget(xcodeProvider)); + return this; + } + + /** + * Adds common xcode settings to the given provider builder. + * + * @param objcProvider provider containing all dependencies' information as well as some of this + * rule's + * @param productType type of this rule's Xcode target + * + * @return this xcode support + */ + XcodeSupport addXcodeSettings(XcodeProvider.Builder xcodeProviderBuilder, + ObjcProvider objcProvider, XcodeProductType productType) { + xcodeProviderBuilder + .setLabel(ruleContext.getLabel()) + .setObjcProvider(objcProvider) + .setProductType(productType); + return this; + } + + /** + * Adds dependencies to the given provider builder from the {@code deps} and {@code bundles} + * attributes. + * + * @return this xcode support + */ + XcodeSupport addDependencies(XcodeProvider.Builder xcodeProviderBuilder) { + xcodeProviderBuilder + .addDependencies(ruleContext.getPrerequisites("deps", Mode.TARGET, XcodeProvider.class)) + .addDependencies(ruleContext.getPrerequisites("bundles", Mode.TARGET, XcodeProvider.class)); + return this; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XibFiles.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XibFiles.java new file mode 100644 index 0000000000..9be1d06a01 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XibFiles.java @@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; + +/** + * A sequence of xib source files. Each {@code .xib} file can be compiled to a {@code .nib} file or + * directory. Because it might be a directory, we always use zip files to store the output and use + * the {@code actooloribtoolzip} utility to run ibtool and zip the output. + */ +public final class XibFiles extends IterableWrapper<Artifact> { + public XibFiles(Iterable<Artifact> artifacts) { + super(artifacts); + } + + /** + * Returns a sequence where each element of this sequence is converted to the file which contains + * the compiled contents of the xib. + */ + public ImmutableList<Artifact> compiledZips(IntermediateArtifacts intermediateArtifacts) { + ImmutableList.Builder<Artifact> zips = new ImmutableList.Builder<>(); + for (Artifact xib : this) { + zips.add(intermediateArtifacts.compiledXibFileZip(xib)); + } + return zips.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoSourcesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoSourcesProvider.java new file mode 100644 index 0000000000..0663f62af4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoSourcesProvider.java @@ -0,0 +1,66 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.proto; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Configured target classes that implement this class can contribute .proto files to the + * compilation of proto_library rules. + */ +@Immutable +public final class ProtoSourcesProvider implements TransitiveInfoProvider { + + private final NestedSet<Artifact> transitiveImports; + private final NestedSet<Artifact> transitiveProtoSources; + private final ImmutableList<Artifact> protoSources; + + public ProtoSourcesProvider(NestedSet<Artifact> transitiveImports, + NestedSet<Artifact> transitiveProtoSources, + ImmutableList<Artifact> protoSources) { + this.transitiveImports = transitiveImports; + this.transitiveProtoSources = transitiveProtoSources; + this.protoSources = protoSources; + } + + /** + * Transitive imports including weak dependencies + * This determines the order of "-I" arguments to the protocol compiler, and + * that is probably important + */ + public NestedSet<Artifact> getTransitiveImports() { + return transitiveImports; + } + + /** + * Returns the proto sources for this rule and all its dependent protocol + * buffer rules. + */ + public NestedSet<Artifact> getTransitiveProtoSources() { + return transitiveProtoSources; + } + + /** + * Returns the proto sources from the 'srcs' attribute. If the library is a proxy library + * that has no sources, return the sources from the direct deps. + */ + public ImmutableList<Artifact> getProtoSources() { + return protoSources; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageAction.java b/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageAction.java new file mode 100644 index 0000000000..6a19f92def --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageAction.java @@ -0,0 +1,132 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Util; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Fingerprint; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +/** + * Generates baseline (empty) coverage for the given non-test target. + */ +public class BaselineCoverageAction extends AbstractFileWriteAction + implements NotifyOnActionCacheHit { + // TODO(bazel-team): Remove this list of languages by separately collecting offline and online + // instrumented files. + private static final List<String> OFFLINE_INSTRUMENTATION_SUFFIXES = ImmutableList.of( + ".c", ".cc", ".cpp", ".dart", ".go", ".h", ".java", ".py"); + private final Iterable<Artifact> instrumentedFiles; + + private BaselineCoverageAction( + ActionOwner owner, Iterable<Artifact> instrumentedFiles, Artifact output) { + super(owner, ImmutableList.<Artifact>of(), output, false); + this.instrumentedFiles = instrumentedFiles; + } + + @Override + public String getMnemonic() { + return "BaselineCoverage"; + } + + @Override + public String computeKey() { + return new Fingerprint() + .addStrings(getInstrumentedFilePathStrings()) + .hexDigestAndReset(); + } + + private Iterable<String> getInstrumentedFilePathStrings() { + List<String> result = new ArrayList<>(); + for (Artifact instrumentedFile : instrumentedFiles) { + String pathString = instrumentedFile.getExecPathString(); + for (String suffix : OFFLINE_INSTRUMENTATION_SUFFIXES) { + if (pathString.endsWith(suffix)) { + result.add(pathString); + break; + } + } + } + + return result; + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, + Executor executor) { + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + PrintWriter writer = new PrintWriter(out); + for (String execPath : getInstrumentedFilePathStrings()) { + writer.write("SF:" + execPath + "\n"); + writer.write("end_of_record\n"); + } + writer.flush(); + } + }; + } + + @Override + protected void afterWrite(Executor executor) { + notifyAboutBaselineCoverage(executor.getEventBus()); + } + + @Override + public void actionCacheHit(Executor executor) { + notifyAboutBaselineCoverage(executor.getEventBus()); + } + + /** + * Notify interested parties about new baseline coverage data. + */ + private void notifyAboutBaselineCoverage(EventBus eventBus) { + Artifact output = Iterables.getOnlyElement(getOutputs()); + String ownerString = Label.print(getOwner().getLabel()); + eventBus.post(new BaselineCoverageResult(output, ownerString)); + } + + /** + * Returns collection of baseline coverage artifacts associated with the given target. + * Will always return 0 or 1 elements. + */ + public static ImmutableList<Artifact> getBaselineCoverageArtifacts(RuleContext ruleContext, + Iterable<Artifact> instrumentedFiles) { + // Baseline coverage artifacts will still go into "testlogs" directory. + Artifact coverageData = ruleContext.getAnalysisEnvironment().getDerivedArtifact( + Util.getWorkspaceRelativePath(ruleContext.getTarget()).getChild("baseline_coverage.dat"), + ruleContext.getConfiguration().getTestLogsDirectory()); + ruleContext.registerAction(new BaselineCoverageAction( + ruleContext.getActionOwner(), instrumentedFiles, coverageData)); + + return ImmutableList.of(coverageData); + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageResult.java b/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageResult.java new file mode 100644 index 0000000000..4af2df00e2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageResult.java @@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; + +/** + * This event is used to notify about a successfully built baseline coverage artifact. + */ +public class BaselineCoverageResult { + + private final Artifact baselineCoverageData; + private final String ownerString; + + public BaselineCoverageResult(Artifact baselineCoverageData, String ownerString) { + this.baselineCoverageData = Preconditions.checkNotNull(baselineCoverageData); + this.ownerString = Preconditions.checkNotNull(ownerString); + } + + public Artifact getArtifact() { + return baselineCoverageData; + } + + public String getOwnerString() { + return ownerString; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/CoverageReportActionFactory.java b/src/main/java/com/google/devtools/build/lib/rules/test/CoverageReportActionFactory.java new file mode 100644 index 0000000000..5f7571a02e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/CoverageReportActionFactory.java @@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; + +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A factory class to create coverage report actions. + */ +public interface CoverageReportActionFactory { + + /** + * Returns a coverage report Action. May return null if it's not necessary to create + * such an Action based on the input parameters and some other data available to + * the factory implementation, such as command line arguments. + */ + @Nullable + public Action createCoverageReportAction(Iterable<ConfiguredTarget> targetsToTest, + Set<Artifact> baselineCoverageArtifacts, + ArtifactFactory artifactFactory, ArtifactOwner artifactOwner); +}
\ No newline at end of file diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/ExclusiveTestStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/test/ExclusiveTestStrategy.java new file mode 100644 index 0000000000..3cb575056b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/ExclusiveTestStrategy.java @@ -0,0 +1,55 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.test; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.view.test.TestStatus.TestResultData; + +import java.io.IOException; + +/** + * Test strategy wrapper called 'exclusive'. It should delegate to a test strategy for local + * execution. The name 'exclusive' triggers behavior it triggers behavior in + * SkyframeExecutor to schedule test execution sequentially after non-test actions. This + * ensures streamed test output is not polluted by other action output. + */ +@ExecutionStrategy(contextType = TestActionContext.class, + name = { "exclusive" }) +public class ExclusiveTestStrategy implements TestActionContext { + private TestActionContext parent; + + public ExclusiveTestStrategy(TestActionContext parent) { + this.parent = parent; + } + + @Override + public void exec(TestRunnerAction action, + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException { + parent.exec(action, actionExecutionContext); + } + + @Override + public TestResult newCachedTestResult( + Path execRoot, TestRunnerAction action, TestResultData cached) throws IOException { + return parent.newCachedTestResult(execRoot, action, cached); + } + + @Override + public String strategyLocality(TestRunnerAction testRunnerAction) { + return "exclusive"; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/ExecutionInfoProvider.java b/src/main/java/com/google/devtools/build/lib/rules/test/ExecutionInfoProvider.java new file mode 100644 index 0000000000..6c0d73cf0c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/ExecutionInfoProvider.java @@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.test; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.Map; + +/** + * This provider can be implemented by rules which need special environments to run in (especially + * tests). + */ +@Immutable +public final class ExecutionInfoProvider implements TransitiveInfoProvider { + + private final ImmutableMap<String, String> executionInfo; + + public ExecutionInfoProvider(Map<String, String> requirements) { + this.executionInfo = ImmutableMap.copyOf(requirements); + } + + /** + * Returns a map to indicate special execution requirements, such as hardware + * platforms, web browsers, etc. Rule tags, such as "requires-XXX", may also be added + * as keys to the map. + */ + public ImmutableMap<String, String> getExecutionInfo() { + return executionInfo; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFileManifestAction.java b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFileManifestAction.java new file mode 100644 index 0000000000..e5ab219cc5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFileManifestAction.java @@ -0,0 +1,133 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Util; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.RegexFilter; +import com.google.devtools.build.lib.vfs.FileSystemUtils; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.Arrays; +import java.util.Collection; + +/** + * Creates instrumented file manifest to list instrumented source files. + */ +class InstrumentedFileManifestAction extends AbstractFileWriteAction { + + private static final String GUID = "d9ddb800-f9a1-01Da-238d-988311a8475b"; + + private final Collection<Artifact> collectedSourceFiles; + private final Collection<Artifact> metadataFiles; + private final RegexFilter instrumentationFilter; + + private InstrumentedFileManifestAction(ActionOwner owner, Collection<Artifact> inputs, + Collection<Artifact> additionalSourceFiles, Collection<Artifact> gcnoFiles, + Artifact output, RegexFilter instrumentationFilter) { + super(owner, inputs, output, false); + this.collectedSourceFiles = additionalSourceFiles; + this.metadataFiles = gcnoFiles; + this.instrumentationFilter = instrumentationFilter; + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) { + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + Writer writer = null; + try { + // Save exec paths for both instrumented source files and gcno files in the manifest + // in the naturally sorted order. + String[] fileNames = Iterables.toArray(Iterables.transform( + Iterables.concat(collectedSourceFiles, metadataFiles), + new Function<Artifact, String> () { + @Override + public String apply(Artifact artifact) { return artifact.getExecPathString(); } + }), String.class); + Arrays.sort(fileNames); + writer = new OutputStreamWriter(out, ISO_8859_1); + for (String name : fileNames) { + writer.write(name); + writer.write('\n'); + } + } finally { + if (writer != null) { + writer.close(); + } + } + } + }; + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addString(instrumentationFilter.toString()); + return f.hexDigestAndReset(); + } + + /** + * Instantiates instrumented file manifest for the given target. + * + * @param ruleContext context of the executable configured target + * @param additionalSourceFiles additional instrumented source files, as + * collected by the {@link InstrumentedFilesCollector} + * @param metadataFiles *.gcno/*.em files collected by the {@link InstrumentedFilesCollector} + * @return instrumented file manifest artifact + */ + public static Artifact getInstrumentedFileManifest(final RuleContext ruleContext, + final Collection<Artifact> additionalSourceFiles, final Collection<Artifact> metadataFiles) { + // Instrumented manifest makes sense only for rules with binary output. + Preconditions.checkState(ruleContext.getRule().hasBinaryOutput()); + final Artifact instrumentedFileManifest = + ruleContext.getAnalysisEnvironment().getDerivedArtifact( + // Do not use replaceExtension(), as we may get name conflicts (two target-names have the + // same base name and only differ by extension). + FileSystemUtils.appendExtension( + Util.getWorkspaceRelativePath(ruleContext.getTarget()), ".instrumented_files"), + ruleContext.getConfiguration().getBinDirectory()); + + // Instrumented manifest artifact might already exist in case when multiple test + // actions that use slightly different subsets of runfiles set are generated for the same rule. + // So check whether we need to create a new action instance. + ImmutableList<Artifact> inputs = ImmutableList.<Artifact>builder() + .addAll(additionalSourceFiles) + .addAll(metadataFiles) + .build(); + ruleContext.registerAction(new InstrumentedFileManifestAction( + ruleContext.getActionOwner(), inputs, additionalSourceFiles, metadataFiles, + instrumentedFileManifest, ruleContext.getConfiguration().getInstrumentationFilter())); + + return instrumentedFileManifest; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesCollector.java b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesCollector.java new file mode 100644 index 0000000000..e62a3b86af --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesCollector.java @@ -0,0 +1,211 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.test; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * A helper class for collecting instrumented files and metadata for a target. + */ +public final class InstrumentedFilesCollector { + + /** + * The set of file types and attributes to visit to collect instrumented files for a certain rule + * type. The class is intentionally immutable, so that a single instance is sufficient for all + * rules of the same type (and in some cases all rules of related types, such as all {@code foo_*} + * rules). + */ + @Immutable + public static final class InstrumentationSpec { + private final FileTypeSet instrumentedFileTypes; + private final Collection<String> instrumentedAttributes; + + public InstrumentationSpec(FileTypeSet instrumentedFileTypes, + Collection<String> instrumentedAttributes) { + this.instrumentedFileTypes = instrumentedFileTypes; + this.instrumentedAttributes = ImmutableList.copyOf(instrumentedAttributes); + } + + public InstrumentationSpec(FileTypeSet instrumentedFileTypes, + String... instrumentedAttributes) { + this(instrumentedFileTypes, ImmutableList.copyOf(instrumentedAttributes)); + } + + /** + * Returns a new instrumentation spec with the given attribute names replacing the ones + * stored in this object. + */ + public InstrumentationSpec withAttributes(String... instrumentedAttributes) { + return new InstrumentationSpec(instrumentedFileTypes, instrumentedAttributes); + } + } + + /** + * The implementation for the local metadata collection. The intention is that implementations + * recurse over the locally (i.e., for that configured target) created actions and collect + * metadata files. + */ + public abstract static class LocalMetadataCollector { + /** + * Recursively runs over the local actions and add metadata files to the metadataFilesBuilder. + */ + public abstract void collectMetadataArtifacts( + Iterable<Artifact> artifacts, AnalysisEnvironment analysisEnvironment, + NestedSetBuilder<Artifact> metadataFilesBuilder); + + /** + * Adds action output of a particular type to metadata files. + * + * <p>Only adds the first output that matches the given file type. + * + * @param metadataFilesBuilder builder to collect metadata files + * @param action the action whose outputs to scan + * @param fileType the filetype of outputs which should be collected + */ + protected void addOutputs(NestedSetBuilder<Artifact> metadataFilesBuilder, + Action action, FileType fileType) { + for (Artifact output : action.getOutputs()) { + if (fileType.matches(output.getFilename())) { + metadataFilesBuilder.add(output); + break; + } + } + } + } + + /** + * Only collects files transitively from srcs, deps, and data attributes. + */ + public static final InstrumentationSpec TRANSITIVE_COLLECTION_SPEC = new InstrumentationSpec( + FileTypeSet.NO_FILE, + "srcs", "deps", "data"); + + /** + * An explicit constant for a {@link LocalMetadataCollector} that doesn't collect anything. + */ + public static final LocalMetadataCollector NO_METADATA_COLLECTOR = null; + + private final RuleContext ruleContext; + private final InstrumentationSpec spec; + private final LocalMetadataCollector localMetadataCollector; + private final NestedSet<Artifact> instrumentationMetadataFiles; + private final NestedSet<Artifact> instrumentedFiles; + + public InstrumentedFilesCollector(RuleContext ruleContext, InstrumentationSpec spec, + LocalMetadataCollector localMetadataCollector, Iterable<Artifact> rootFiles) { + this.ruleContext = ruleContext; + this.spec = spec; + this.localMetadataCollector = localMetadataCollector; + Preconditions.checkNotNull(ruleContext, "RuleContext already cleared. That means that the" + + " collector data was already memoized. You do not have to call it again."); + if (!ruleContext.getConfiguration().isCodeCoverageEnabled()) { + instrumentedFiles = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + instrumentationMetadataFiles = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } else { + NestedSetBuilder<Artifact> instrumentedFilesBuilder = + NestedSetBuilder.stableOrder(); + NestedSetBuilder<Artifact> metadataFilesBuilder = NestedSetBuilder.stableOrder(); + collect(ruleContext.getAnalysisEnvironment(), instrumentedFilesBuilder, metadataFilesBuilder, + rootFiles); + instrumentedFiles = instrumentedFilesBuilder.build(); + instrumentationMetadataFiles = metadataFilesBuilder.build(); + } + } + + /** + * Returns instrumented source files for the target provided during construction. + */ + public final NestedSet<Artifact> getInstrumentedFiles() { + return instrumentedFiles; + } + + /** + * Returns instrumentation metadata files for the target provided during construction. + */ + public final NestedSet<Artifact> getInstrumentationMetadataFiles() { + return instrumentationMetadataFiles; + } + + /** + * Collects instrumented files and metadata files. + */ + private void collect(AnalysisEnvironment analysisEnvironment, + NestedSetBuilder<Artifact> instrumentedFilesBuilder, + NestedSetBuilder<Artifact> metadataFilesBuilder, + Iterable<Artifact> rootFiles) { + for (TransitiveInfoCollection dep : getAllPrerequisites()) { + InstrumentedFilesProvider provider = dep.getProvider(InstrumentedFilesProvider.class); + if (provider != null) { + instrumentedFilesBuilder.addTransitive(provider.getInstrumentedFiles()); + metadataFilesBuilder.addTransitive(provider.getInstrumentationMetadataFiles()); + } else if (shouldIncludeLocalSources()) { + for (Artifact artifact : dep.getProvider(FileProvider.class).getFilesToBuild()) { + if (artifact.isSourceArtifact() && + spec.instrumentedFileTypes.matches(artifact.getFilename())) { + instrumentedFilesBuilder.add(artifact); + } + } + } + } + + if (localMetadataCollector != null) { + localMetadataCollector.collectMetadataArtifacts(rootFiles, + analysisEnvironment, metadataFilesBuilder); + } + } + + /** + * Returns the list of attributes which should be (transitively) checked for sources and + * instrumentation metadata. + */ + private Collection<String> getSourceAttributes() { + return spec.instrumentedAttributes; + } + + private boolean shouldIncludeLocalSources() { + return ruleContext.getConfiguration().getInstrumentationFilter().isIncluded( + ruleContext.getLabel().toString()); + } + + private Iterable<TransitiveInfoCollection> getAllPrerequisites() { + List<TransitiveInfoCollection> prerequisites = new ArrayList<>(); + for (String attr : getSourceAttributes()) { + if (ruleContext.getRule().isAttrDefined(attr, Type.LABEL_LIST) || + ruleContext.getRule().isAttrDefined(attr, Type.LABEL)) { + Iterables.addAll(prerequisites, ruleContext.getPrerequisites(attr, Mode.DONT_CHECK)); + } + } + return prerequisites; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProvider.java new file mode 100644 index 0000000000..b1f956c54c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProvider.java @@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; + +/** + * A provider of instrumented file sources and instrumentation metadata. + */ +public interface InstrumentedFilesProvider extends TransitiveInfoProvider { + + /** + * Returns a collection of source files for instrumented binaries. + */ + NestedSet<Artifact> getInstrumentedFiles(); + + /** + * Returns a collection of instrumentation metadata files. + */ + NestedSet<Artifact> getInstrumentationMetadataFiles(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProviderImpl.java b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProviderImpl.java new file mode 100644 index 0000000000..1452e2db43 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProviderImpl.java @@ -0,0 +1,53 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.test; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; + +/** + * An implementation class for the InstrumentedFilesProvider interface. + */ +public final class InstrumentedFilesProviderImpl implements InstrumentedFilesProvider { + public static final InstrumentedFilesProvider EMPTY = new InstrumentedFilesProvider() { + @Override + public NestedSet<Artifact> getInstrumentedFiles() { + return NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER); + } + @Override + public NestedSet<Artifact> getInstrumentationMetadataFiles() { + return NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER); + } + }; + + private final NestedSet<Artifact> instrumentedFiles; + private final NestedSet<Artifact> instrumentationMetadataFiles; + + public InstrumentedFilesProviderImpl(InstrumentedFilesCollector collector) { + this.instrumentedFiles = collector.getInstrumentedFiles(); + this.instrumentationMetadataFiles = collector.getInstrumentationMetadataFiles(); + } + + @Override + public NestedSet<Artifact> getInstrumentedFiles() { + return instrumentedFiles; + } + + @Override + public NestedSet<Artifact> getInstrumentationMetadataFiles() { + return instrumentationMetadataFiles; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java new file mode 100644 index 0000000000..006f789599 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java @@ -0,0 +1,224 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.BaseSpawn; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.actions.TestExecException; +import com.google.devtools.build.lib.analysis.config.BinTools; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; +import com.google.devtools.build.lib.view.test.TestStatus.TestCase; +import com.google.devtools.build.lib.view.test.TestStatus.TestResultData; +import com.google.devtools.common.options.OptionsClassProvider; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Runs TestRunnerAction actions. + */ +@ExecutionStrategy(contextType = TestActionContext.class, + name = { "standalone" }) +public class StandaloneTestStrategy extends TestStrategy { + /* + TODO(bazel-team): + + * tests + * It would be nice to get rid of (cd $TEST_SRCDIR) in the test-setup script. + * test timeouts. + * parsing XML output. + + */ + protected final PathFragment runfilesPrefix; + + public StandaloneTestStrategy(OptionsClassProvider requestOptions, + OptionsClassProvider startupOptions, BinTools binTools, PathFragment runfilesPrefix) { + super(requestOptions, startupOptions, binTools); + + this.runfilesPrefix = runfilesPrefix; + } + + private static final String TEST_SETUP = "tools/test/test-setup.sh"; + + @Override + public void exec(TestRunnerAction action, ActionExecutionContext actionExecutionContext) + throws ExecException, InterruptedException { + Path runfilesDir = null; + try { + runfilesDir = TestStrategy.getLocalRunfilesDirectory( + action, actionExecutionContext, binTools); + } catch (ExecException e) { + throw new TestExecException(e.getMessage()); + } + + Path workingDirectory = runfilesDir.getRelative(runfilesPrefix); + Map<String, String> env = getEnv(action, runfilesDir); + Spawn spawn = new BaseSpawn(getArgs(action), env, + action.getTestProperties().getExecutionInfo(), + action, + action.getTestProperties().getLocalResourceUsage()); + + Executor executor = actionExecutionContext.getExecutor(); + try { + FileSystemUtils.createDirectoryAndParents(workingDirectory); + FileOutErr fileOutErr = new FileOutErr(action.getTestLog().getPath(), + action.resolve(actionExecutionContext.getExecutor().getExecRoot()).getTestStderr()); + TestResultData data = execute( + actionExecutionContext.withFileOutErr(fileOutErr), spawn, action); + appendStderr(fileOutErr.getOutputFile(), fileOutErr.getErrorFile()); + finalizeTest(actionExecutionContext, action, data); + } catch (IOException e) { + executor.getEventHandler().handle(Event.error("Caught I/O exception: " + e)); + throw new EnvironmentalExecException("unexpected I/O exception", e); + } + } + + private Map<String, String> getEnv(TestRunnerAction action, Path runfilesDir) { + Map<String, String> vars = getDefaultTestEnvironment(action); + BuildConfiguration config = action.getConfiguration(); + + vars.putAll(config.getDefaultShellEnvironment()); + vars.putAll(config.getTestEnv()); + vars.put("TEST_SRCDIR", runfilesDir.getRelative(runfilesPrefix).getPathString()); + + // TODO(bazel-team): set TEST_TMPDIR. + + return vars; + } + + private TestResultData execute( + ActionExecutionContext actionExecutionContext, Spawn spawn, TestRunnerAction action) + throws TestExecException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + Closeable streamed = null; + Path testLogPath = action.getTestLog().getPath(); + TestResultData.Builder builder = TestResultData.newBuilder(); + + try { + try { + if (executionOptions.testOutput.equals(TestOutputFormat.STREAMED)) { + streamed = new StreamedTestOutput( + Reporter.outErrForReporter( + actionExecutionContext.getExecutor().getEventHandler()), testLogPath); + } + executor.getSpawnActionContext(action.getMnemonic()).exec(spawn, actionExecutionContext); + + builder.setTestPassed(true) + .setStatus(BlazeTestStatus.PASSED) + .setCachable(true); + } catch (ExecException e) { + // Execution failed, which we consider a test failure. + + // TODO(bazel-team): set cachable==true for relevant statuses (failure, but not for + // timeout, etc.) + builder.setTestPassed(false) + .setStatus(BlazeTestStatus.FAILED); + } finally { + if (streamed != null) { + streamed.close(); + } + } + + TestCase details = parseTestResult( + action.resolve(actionExecutionContext.getExecutor().getExecRoot()).getXmlOutputPath()); + if (details != null) { + builder.setTestCase(details); + } + + return builder.build(); + } catch (IOException e) { + throw new TestExecException(e.getMessage()); + } + } + + /** + * Outputs test result to the stdout after test has finished (e.g. for --test_output=all or + * --test_output=errors). Will also try to group output lines together (up to 10000 lines) so + * parallel test outputs will not get interleaved. + */ + protected void processTestOutput(Executor executor, FileOutErr outErr, TestResult result) + throws IOException { + Path testOutput = executor.getExecRoot().getRelative(result.getTestLogPath().asFragment()); + boolean isPassed = result.getData().getTestPassed(); + try { + if (TestLogHelper.shouldOutputTestLog(executionOptions.testOutput, isPassed)) { + TestLogHelper.writeTestLog(testOutput, result.getTestName(), outErr.getOutputStream()); + } + } finally { + if (isPassed) { + executor.getEventHandler().handle(new Event(EventKind.PASS, null, result.getTestName())); + } else { + if (result.getData().getStatus() == BlazeTestStatus.TIMEOUT) { + executor.getEventHandler().handle( + new Event(EventKind.TIMEOUT, null, result.getTestName() + + " (see " + testOutput + ")")); + } else { + executor.getEventHandler().handle( + new Event(EventKind.FAIL, null, result.getTestName() + " (see " + testOutput + ")")); + } + } + } + } + + private final void finalizeTest(ActionExecutionContext actionExecutionContext, + TestRunnerAction action, TestResultData data) throws IOException, ExecException { + TestResult result = new TestResult(action, data, false); + postTestResult(actionExecutionContext.getExecutor(), result); + + processTestOutput(actionExecutionContext.getExecutor(), + actionExecutionContext.getFileOutErr(), result); + // TODO(bazel-team): handle --test_output=errors, --test_output=all. + + if (!executionOptions.testKeepGoing && data.getStatus() != BlazeTestStatus.PASSED) { + throw new TestExecException("Test failed: aborting"); + } + } + + private List<String> getArgs(TestRunnerAction action) { + List<String> args = Lists.newArrayList(TEST_SETUP); + TestTargetExecutionSettings execSettings = action.getExecutionSettings(); + + // Execute the test using the alias in the runfiles tree. + args.add(execSettings.getExecutable().getRootRelativePath().getPathString()); + args.addAll(execSettings.getArgs()); + + return args; + } + + @Override + public String strategyLocality(TestRunnerAction action) { return "standalone"; } + + @Override + public TestResult newCachedTestResult( + Path execRoot, TestRunnerAction action, TestResultData data) { + return new TestResult(action, data, /*cached*/ true); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestActionBuilder.java new file mode 100644 index 0000000000..2ac9a0fc5f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestActionBuilder.java @@ -0,0 +1,270 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.Util; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.TestSize; +import com.google.devtools.build.lib.packages.TestTimeout; +import com.google.devtools.build.lib.rules.test.TestProvider.TestParams; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.EnumConverter; + +import java.util.Collection; +import java.util.List; + +import javax.annotation.Nullable; + +/** + * Helper class to create test actions. + */ +public final class TestActionBuilder { + + private final RuleContext ruleContext; + private RunfilesSupport runfilesSupport; + private Artifact executable; + private ExecutionInfoProvider executionRequirements; + private InstrumentedFilesProvider instrumentedFiles; + private int explicitShardCount; + + public TestActionBuilder(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + /** + * Creates the test actions and artifacts using the previously set parameters. + * + * @return ordered list of test status artifacts + */ + public TestParams build() { + Preconditions.checkState(runfilesSupport != null); + boolean local = TargetUtils.isTestRuleAndRunsLocally(ruleContext.getRule()); + TestShardingStrategy strategy = ruleContext.getConfiguration().testShardingStrategy(); + int shards = strategy.getNumberOfShards( + local, explicitShardCount, isTestShardingCompliant(), + TestSize.getTestSize(ruleContext.getRule())); + Preconditions.checkState(shards >= 0); + return createTestAction(Util.getWorkspaceRelativePath(ruleContext.getLabel()), shards); + } + + private boolean isTestShardingCompliant() { + // See if it has a data dependency on the special target + // //tools:test_sharding_compliant. Test runners add this dependency + // to show they speak the sharding protocol. + // There are certain cases where this heuristic may fail, giving + // a "false positive" (where we shard the test even though the + // it isn't supported). We may want to refine this logic, but + // heuristically sharding is currently experimental. Also, we do detect + // false-positive cases and return an error. + return runfilesSupport.getRunfilesSymlinkNames().contains( + new PathFragment("tools/test_sharding_compliant")); + } + + /** + * Set the runfiles and executable to be run as a test. + */ + public TestActionBuilder setFilesToRunProvider(FilesToRunProvider provider) { + Preconditions.checkNotNull(provider.getRunfilesSupport()); + Preconditions.checkNotNull(provider.getExecutable()); + this.runfilesSupport = provider.getRunfilesSupport(); + this.executable = provider.getExecutable(); + return this; + } + + public TestActionBuilder setInstrumentedFiles( + @Nullable InstrumentedFilesProvider instrumentedFiles) { + this.instrumentedFiles = instrumentedFiles; + return this; + } + + public TestActionBuilder setExecutionRequirements( + @Nullable ExecutionInfoProvider executionRequirements) { + this.executionRequirements = executionRequirements; + return this; + } + + /** + * Set the explicit shard count. Note that this may be overridden by the sharding strategy. + */ + public TestActionBuilder setShardCount(int explicitShardCount) { + this.explicitShardCount = explicitShardCount; + return this; + } + + /** + * Converts to {@link TestActionBuilder.TestShardingStrategy}. + */ + public static class ShardingStrategyConverter extends EnumConverter<TestShardingStrategy> { + public ShardingStrategyConverter() { + super(TestShardingStrategy.class, "test sharding strategy"); + } + } + + /** + * A strategy for running the same tests in many processes. + */ + public static enum TestShardingStrategy { + EXPLICIT { + @Override public int getNumberOfShards(boolean isLocal, int shardCountFromAttr, + boolean testShardingCompliant, TestSize testSize) { + return Math.max(shardCountFromAttr, 0); + } + }, + + EXPERIMENTAL_HEURISTIC { + @Override public int getNumberOfShards(boolean isLocal, int shardCountFromAttr, + boolean testShardingCompliant, TestSize testSize) { + if (shardCountFromAttr >= 0) { + return shardCountFromAttr; + } + if (isLocal || !testShardingCompliant) { + return 0; + } + return testSize.getDefaultShards(); + } + }, + + DISABLED { + @Override public int getNumberOfShards(boolean isLocal, int shardCountFromAttr, + boolean testShardingCompliant, TestSize testSize) { + return 0; + } + }; + + public abstract int getNumberOfShards(boolean isLocal, int shardCountFromAttr, + boolean testShardingCompliant, TestSize testSize); + } + + /** + * Creates a test action and artifacts for the given rule. The test action will + * use the specified executable and runfiles. + * + * @param targetName the relative path of the target to run + * @return ordered list of test artifacts, one per action. These are used to drive + * execution in Skyframe, and by AggregatingTestListener and + * TestResultAnalyzer to keep track of completed and pending test runs. + */ + private TestParams createTestAction(PathFragment targetName, int shards) { + BuildConfiguration config = ruleContext.getConfiguration(); + AnalysisEnvironment env = ruleContext.getAnalysisEnvironment(); + Root root = config.getTestLogsDirectory(); + + NestedSetBuilder<Artifact> inputsBuilder = NestedSetBuilder.stableOrder(); + inputsBuilder.addTransitive( + NestedSetBuilder.create(Order.STABLE_ORDER, runfilesSupport.getRunfilesMiddleman())); + for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("$test_runtime", Mode.HOST)) { + inputsBuilder.addTransitive(dep.getProvider(FileProvider.class).getFilesToBuild()); + } + TestTargetProperties testProperties = new TestTargetProperties( + ruleContext, executionRequirements); + + // If the test rule does not provide InstrumentedFilesProvider, there's not much that we can do. + final boolean collectCodeCoverage = config.isCodeCoverageEnabled() + && instrumentedFiles != null; + + TestTargetExecutionSettings executionSettings; + if (collectCodeCoverage) { + // Add instrumented file manifest artifact to the list of inputs. This file will contain + // exec paths of all source files that should be included into the code coverage output. + Collection<Artifact> metadataFiles = + ImmutableList.copyOf(instrumentedFiles.getInstrumentationMetadataFiles()); + inputsBuilder.addTransitive(NestedSetBuilder.wrap(Order.STABLE_ORDER, metadataFiles)); + for (TransitiveInfoCollection dep : + ruleContext.getPrerequisites(":coverage_support", Mode.HOST)) { + inputsBuilder.addTransitive(dep.getProvider(FileProvider.class).getFilesToBuild()); + } + Artifact instrumentedFileManifest = + InstrumentedFileManifestAction.getInstrumentedFileManifest(ruleContext, + ImmutableList.copyOf(instrumentedFiles.getInstrumentedFiles()), + metadataFiles); + executionSettings = new TestTargetExecutionSettings(ruleContext, runfilesSupport, + executable, instrumentedFileManifest, shards); + inputsBuilder.add(instrumentedFileManifest); + } else { + executionSettings = new TestTargetExecutionSettings(ruleContext, runfilesSupport, + executable, null, shards); + } + + if (config.getRunUnder() != null) { + Artifact runUnderExecutable = executionSettings.getRunUnderExecutable(); + if (runUnderExecutable != null) { + inputsBuilder.add(runUnderExecutable); + } + } + + int runsPerTest = config.getRunsPerTestForLabel(ruleContext.getLabel()); + + Iterable<Artifact> inputs = inputsBuilder.build(); + int shardRuns = (shards > 0 ? shards : 1); + List<Artifact> results = Lists.newArrayListWithCapacity(runsPerTest * shardRuns); + ImmutableList.Builder<Artifact> coverageArtifacts = ImmutableList.builder(); + + for (int run = 0; run < runsPerTest; run++) { + // Use a 1-based index for user friendliness. + String runSuffix = + runsPerTest > 1 ? String.format("_run_%d_of_%d", run + 1, runsPerTest) : ""; + for (int shard = 0; shard < shardRuns; shard++) { + String suffix = (shardRuns > 1 ? String.format("_shard_%d_of_%d", shard + 1, shards) : "") + + runSuffix; + Artifact testLog = env.getDerivedArtifact( + targetName.getChild("test" + suffix + ".log"), root); + Artifact cacheStatus = env.getDerivedArtifact( + targetName.getChild("test" + suffix + ".cache_status"), root); + + Artifact coverageArtifact = null; + if (collectCodeCoverage) { + coverageArtifact = + env.getDerivedArtifact(targetName.getChild("coverage" + suffix + ".dat"), root); + coverageArtifacts.add(coverageArtifact); + } + + Artifact microCoverageArtifact = null; + if (collectCodeCoverage && config.isMicroCoverageEnabled()) { + microCoverageArtifact = + env.getDerivedArtifact(targetName.getChild("coverage" + suffix + ".micro.dat"), root); + } + + env.registerAction(new TestRunnerAction( + ruleContext.getActionOwner(), inputs, + testLog, cacheStatus, + coverageArtifact, microCoverageArtifact, + testProperties, executionSettings, + shard, run, config)); + results.add(cacheStatus); + } + } + // TODO(bazel-team): Passing the reportGenerator to every TestParams is a bit strange. + Artifact reportGenerator = collectCodeCoverage + ? ruleContext.getPrerequisiteArtifact(":coverage_report_generator", Mode.HOST) : null; + return new TestParams(runsPerTest, shards, TestTimeout.getTestTimeout(ruleContext.getRule()), + ruleContext.getRule().getRuleClass(), ImmutableList.copyOf(results), + coverageArtifacts.build(), reportGenerator); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestActionContext.java new file mode 100644 index 0000000000..7f7a916a57 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestActionContext.java @@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.test; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.view.test.TestStatus.TestResultData; + +import java.io.IOException; + +/** + * A context for the execution of test actions ({@link TestRunnerAction}). + */ +public interface TestActionContext extends ActionContext { + + /** + * Executes the test command, directing standard out / err to {@code outErr}. The status of + * the test should be communicated by posting a {@link TestResult} object to the eventbus. + */ + void exec(TestRunnerAction action, + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException; + + /** + * String describing where the action will run. + */ + String strategyLocality(TestRunnerAction action); + + /** + * Creates a cached test result. + */ + TestResult newCachedTestResult(Path execRoot, TestRunnerAction action, TestResultData cached) + throws IOException; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestLogHelper.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestLogHelper.java new file mode 100644 index 0000000000..462c24ce6c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestLogHelper.java @@ -0,0 +1,141 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.test; + +import com.google.common.io.ByteStreams; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.BufferedOutputStream; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; + +/** + * A helper class for test log handling. It determines whether the test log should + * be output and formats the test log for console display. + */ +public class TestLogHelper { + + public static final String HEADER_DELIMITER = + "-----------------------------------------------------------------------------"; + + /** + * Determines whether the test log should be output from the current outputMode + * and whether the test has passed or not. + */ + public static boolean shouldOutputTestLog(TestOutputFormat outputMode, boolean hasPassed) { + return (outputMode == TestOutputFormat.ALL) || + (!hasPassed && (outputMode == TestOutputFormat.ERRORS)); + } + + /** + * Reads the contents of the test log from the provided testOutput file, adds + * header and footer and returns the result. + * This method also looks for a header delimiter and cuts off the text before it, + * except if the header is 50 lines or longer. + */ + public static void writeTestLog(Path testOutput, String testName, OutputStream out) + throws IOException { + InputStream input = null; + PrintStream printOut = new PrintStream(new BufferedOutputStream(out)); + try { + final String outputHeader = + "==================== Test output for " + testName + ":"; + final String outputFooter = + "================================================================================"; + + printOut.println(outputHeader); + printOut.flush(); + + input = testOutput.getInputStream(); + FilterTestHeaderOutputStream filteringOutputStream = getHeaderFilteringOutputStream(printOut); + ByteStreams.copy(input, filteringOutputStream); + + if (!filteringOutputStream.foundHeader()) { + InputStream inputAgain = testOutput.getInputStream(); + try { + ByteStreams.copy(inputAgain, out); + } finally { + inputAgain.close(); + } + } + + printOut.println(outputFooter); + } finally { + printOut.flush(); + if (input != null) { + input.close(); + } + } + } + + /** + * Returns an output stream that doesn't write to original until it + * sees HEADER_DELIMITER by itself on a line. + */ + public static FilterTestHeaderOutputStream getHeaderFilteringOutputStream(OutputStream original) { + return new FilterTestHeaderOutputStream(original); + } + + private TestLogHelper() { + // Prevent Java from creating a public constructor. + } + + /** + * Use this class to filter the streaming output of a test until we see the + * header delimiter. + */ + public static class FilterTestHeaderOutputStream extends FilterOutputStream { + + private boolean seenDelimiter = false; + private StringBuilder lineBuilder = new StringBuilder(); + + private static final int NEWLINE = '\n'; + + public FilterTestHeaderOutputStream(OutputStream out) { + super(out); + } + + @Override + public void write(int b) throws IOException { + if (seenDelimiter) { + out.write(b); + } else if (b == NEWLINE) { + String line = lineBuilder.toString(); + lineBuilder = new StringBuilder(); + if (line.equals(TestLogHelper.HEADER_DELIMITER)) { + seenDelimiter = true; + } + } else if (lineBuilder.length() <= TestLogHelper.HEADER_DELIMITER.length()) { + lineBuilder.append((char) b); + } + } + + @Override + public void write(byte b[], int off, int len) throws IOException { + if (seenDelimiter) { + out.write(b, off, len); + } else { + super.write(b, off, len); + } + } + + public boolean foundHeader() { + return seenDelimiter; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestProvider.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestProvider.java new file mode 100644 index 0000000000..d24fe6bb77 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestProvider.java @@ -0,0 +1,143 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.packages.TestTimeout; + +import java.util.List; + +/** + * A {@link TransitiveInfoProvider} for configured targets that implement test rules. + */ +@Immutable +public final class TestProvider implements TransitiveInfoProvider { + private final TestParams testParams; + private final ImmutableList<String> testTags; + + public TestProvider(TestParams testParams, ImmutableList<String> testTags) { + this.testParams = testParams; + this.testTags = testTags; + } + + /** + * Returns the {@link TestParams} object for the test represented by the corresponding configured + * target. + */ + public TestParams getTestParams() { + return testParams; + } + + /** + * Temporary hack to allow dependencies on test_suite targets to continue to work for the time + * being. + */ + public List<String> getTestTags() { + return testTags; + } + + /** + * Returns the test status artifacts for a specified configured target + * + * @param target the configured target. Should belong to a test rule. + * @return the test status artifacts + */ + public static ImmutableList<Artifact> getTestStatusArtifacts(TransitiveInfoCollection target) { + return target.getProvider(TestProvider.class).getTestParams().getTestStatusArtifacts(); + } + + /** + * A value class describing the properties of a test. + */ + public static class TestParams { + private final int runs; + private final int shards; + private final TestTimeout timeout; + private final String testRuleClass; + private final ImmutableList<Artifact> testStatusArtifacts; + private final ImmutableList<Artifact> coverageArtifacts; + private final Artifact coverageReportGenerator; + + /** + * Don't call this directly. Instead use {@link TestActionBuilder}. + */ + TestParams(int runs, int shards, TestTimeout timeout, String testRuleClass, + ImmutableList<Artifact> testStatusArtifacts, + ImmutableList<Artifact> coverageArtifacts, + Artifact coverageReportGenerator) { + this.runs = runs; + this.shards = shards; + this.timeout = timeout; + this.testRuleClass = testRuleClass; + this.testStatusArtifacts = testStatusArtifacts; + this.coverageArtifacts = coverageArtifacts; + this.coverageReportGenerator = coverageReportGenerator; + } + + /** + * Returns the number of times this test should be run. + */ + public int getRuns() { + return runs; + } + + /** + * Returns the number of shards for this test. + */ + public int getShards() { + return shards; + } + + /** + * Returns the timeout of this test. + */ + public TestTimeout getTimeout() { + return timeout; + } + + /** + * Returns the test rule class. + */ + public String getTestRuleClass() { + return testRuleClass; + } + + /** + * Returns a list of test status artifacts that represent serialized test status protobuffers + * produced by testing this target. + */ + public ImmutableList<Artifact> getTestStatusArtifacts() { + return testStatusArtifacts; + } + + /** + * Returns the coverageArtifacts + */ + public ImmutableList<Artifact> getCoverageArtifacts() { + return coverageArtifacts; + } + + /** + * Returns the coverage report generator tool. + */ + public Artifact getCoverageReportGenerator() { + return coverageReportGenerator; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestResult.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestResult.java new file mode 100644 index 0000000000..b0de6cd6e6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestResult.java @@ -0,0 +1,133 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; +import com.google.devtools.build.lib.view.test.TestStatus.TestResultData; + +/** + * This is the event passed from the various test strategies to the {@code RecordingTestListener} + * upon test completion. + */ +@ThreadSafe +@Immutable +public class TestResult { + + private final TestRunnerAction testAction; + private final TestResultData data; + private final boolean cached; + + /** + * Construct the TestResult for the given test / status. + * + * @param testAction The test that was run. + * @param data test result protobuffer. + * @param cached true if this is a cached test result. + */ + public TestResult(TestRunnerAction testAction, TestResultData data, boolean cached) { + this.testAction = Preconditions.checkNotNull(testAction); + this.data = data; + this.cached = cached; + } + + public static boolean isBlazeTestStatusPassed(BlazeTestStatus status) { + return status == BlazeTestStatus.PASSED || status == BlazeTestStatus.FLAKY; + } + + /** + * @return The test action. + */ + public TestRunnerAction getTestAction() { + return testAction; + } + + /** + * @return The test log path. Note, that actual log file may no longer + * correspond to this artifact - use getActualLogPath() method if + * you need log location. + */ + public Path getTestLogPath() { + return testAction.getTestLog().getPath(); + } + + /** + * Return if result was loaded from local action cache. + */ + public final boolean isCached() { + return cached; + } + + /** + * @return Coverage data artifact, if available and null otherwise. + */ + public PathFragment getCoverageData() { + if (data.getHasCoverage()) { + return testAction.getCoverageData().getExecPath(); + } + return null; + } + + /** + * @return The test status artifact. + */ + public Artifact getTestStatusArtifact() { + // these artifacts are used to keep track of the number of pending and completed tests. + return testAction.getCacheStatusArtifact(); + } + + + /** + * Gets the test name in a user-friendly format. + * Will generally include the target name and shard number, if applicable. + * + * @return The test name. + */ + public String getTestName() { + return testAction.getTestName(); + } + + /** + * @return The test label. + */ + public String getLabel() { + return Label.print(testAction.getOwner().getLabel()); + } + + /** + * @return The test shard number. + */ + public int getShardNum() { + return testAction.getShardNum(); + } + + /** + * @return Total number of test shards. 0 means + * no sharding, whereas 1 means degenerate sharding. + */ + public int getTotalShards() { + return testAction.getExecutionSettings().getTotalShards(); + } + + public TestResultData getData() { + return data; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestRunnerAction.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestRunnerAction.java new file mode 100644 index 0000000000..28500a752c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestRunnerAction.java @@ -0,0 +1,607 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.RunUnder; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.test.TestStatus.TestResultData; +import com.google.devtools.common.options.TriState; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.logging.Level; + +import javax.annotation.Nullable; + +/** + * An Action representing a test with the associated environment (runfiles, + * environment variables, test result, etc). It consumes test executable and + * runfiles artifacts and produces test result and test status artifacts. + */ +// Not final so that we can mock it in tests. +public class TestRunnerAction extends AbstractAction implements NotifyOnActionCacheHit { + + private static final String GUID = "94857c93-f11c-4cbc-8c1b-e0a281633f9e"; + + private final BuildConfiguration configuration; + private final Artifact testLog; + private final Artifact cacheStatus; + private final PathFragment testWarningsPath; + private final PathFragment splitLogsPath; + private final PathFragment splitLogsDir; + private final PathFragment undeclaredOutputsDir; + private final PathFragment undeclaredOutputsZipPath; + private final PathFragment undeclaredOutputsAnnotationsDir; + private final PathFragment undeclaredOutputsManifestPath; + private final PathFragment undeclaredOutputsAnnotationsPath; + private final PathFragment xmlOutputPath; + @Nullable + private final PathFragment testShard; + private final PathFragment testExitSafe; + private final PathFragment testStderr; + private final PathFragment testInfrastructureFailure; + private final PathFragment baseDir; + private final String namePrefix; + private final Artifact coverageData; + private final Artifact microCoverageData; + private final TestTargetProperties testProperties; + private final TestTargetExecutionSettings executionSettings; + private final int shardNum; + private final int runNumber; + + // Mutable state related to test caching. + private boolean checkedCaching = false; + private boolean unconditionalExecution = false; + + private static ImmutableList<Artifact> list(Artifact... artifacts) { + ImmutableList.Builder<Artifact> builder = ImmutableList.builder(); + for (Artifact artifact : artifacts) { + if (artifact != null) { + builder.add(artifact); + } + } + return builder.build(); + } + + /** + * Create new TestRunnerAction instance. Should not be called directly. + * Use {@link TestActionBuilder} instead. + * + * @param shardNum The shard number. Must be 0 if totalShards == 0 + * (no sharding). Otherwise, must be >= 0 and < totalShards. + * @param runNumber test run number + */ + TestRunnerAction(ActionOwner owner, + Iterable<Artifact> inputs, + Artifact testLog, + Artifact cacheStatus, + Artifact coverageArtifact, + Artifact microCoverageArtifact, + TestTargetProperties testProperties, + TestTargetExecutionSettings executionSettings, + int shardNum, + int runNumber, + BuildConfiguration configuration) { + super(owner, inputs, list(testLog, cacheStatus, coverageArtifact, microCoverageArtifact)); + this.configuration = Preconditions.checkNotNull(configuration); + this.testLog = testLog; + this.cacheStatus = cacheStatus; + this.coverageData = coverageArtifact; + this.microCoverageData = microCoverageArtifact; + this.shardNum = shardNum; + this.runNumber = runNumber; + this.testProperties = Preconditions.checkNotNull(testProperties); + this.executionSettings = Preconditions.checkNotNull(executionSettings); + + this.baseDir = cacheStatus.getExecPath().getParentDirectory(); + this.namePrefix = FileSystemUtils.removeExtension(cacheStatus.getExecPath().getBaseName()); + + int totalShards = executionSettings.getTotalShards(); + Preconditions.checkState((totalShards == 0 && shardNum == 0) || + (totalShards > 0 && 0 <= shardNum && shardNum < totalShards)); + this.testExitSafe = baseDir.getChild(namePrefix + ".exited_prematurely"); + // testShard Path should be set only if sharding is enabled. + this.testShard = totalShards > 1 + ? baseDir.getChild(namePrefix + ".shard") + : null; + this.xmlOutputPath = baseDir.getChild(namePrefix + ".xml"); + this.testWarningsPath = baseDir.getChild(namePrefix + ".warnings"); + this.testStderr = baseDir.getChild(namePrefix + ".err"); + this.splitLogsDir = baseDir.getChild(namePrefix + ".raw_splitlogs"); + // See note in {@link #getSplitLogsPath} on the choice of file name. + this.splitLogsPath = splitLogsDir.getChild("test.splitlogs"); + this.undeclaredOutputsDir = baseDir.getChild(namePrefix + ".outputs"); + this.undeclaredOutputsZipPath = undeclaredOutputsDir.getChild("outputs.zip"); + this.undeclaredOutputsAnnotationsDir = baseDir.getChild(namePrefix + ".outputs_manifest"); + this.undeclaredOutputsManifestPath = undeclaredOutputsAnnotationsDir.getChild("MANIFEST"); + this.undeclaredOutputsAnnotationsPath = undeclaredOutputsAnnotationsDir.getChild("ANNOTATIONS"); + this.testInfrastructureFailure = baseDir.getChild(namePrefix + ".infrastructure_failure"); + } + + public BuildConfiguration getConfiguration() { + return configuration; + } + + public final PathFragment getBaseDir() { + return baseDir; + } + + public final String getNamePrefix() { + return namePrefix; + } + + @Override + public boolean showsOutputUnconditionally() { + return true; + } + + @Override + public int getInputCount() { + return Iterables.size(getInputs()); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addStrings(executionSettings.getArgs()); + f.addString(executionSettings.getTestFilter() == null ? "" : executionSettings.getTestFilter()); + RunUnder runUnder = executionSettings.getRunUnder(); + f.addString(runUnder == null ? "" : runUnder.getValue()); + f.addStringMap(configuration.getTestEnv()); + f.addString(testProperties.getSize().toString()); + f.addString(testProperties.getTimeout().toString()); + f.addStrings(testProperties.getTags()); + f.addInt(testProperties.isLocal() ? 1 : 0); + f.addInt(shardNum); + f.addInt(executionSettings.getTotalShards()); + f.addInt(runNumber); + f.addInt(configuration.getRunsPerTestForLabel(getOwner().getLabel())); + f.addInt(configuration.isCodeCoverageEnabled() ? 1 : 0); + return f.hexDigestAndReset(); + } + + @Override + public boolean executeUnconditionally() { + // Note: isVolatile must return true if executeUnconditionally can ever return true + // for this instance. + unconditionalExecution = updateExecuteUnconditionallyFromTestStatus(); + checkedCaching = true; + return unconditionalExecution; + } + + @Override + public boolean isVolatile() { + return true; + } + + /** + * Saves cache status to disk. + */ + public void saveCacheStatus(TestResultData data) throws IOException { + try (OutputStream out = cacheStatus.getPath().getOutputStream()) { + data.writeTo(out); + } + } + + /** + * Returns the cache from disk, or null if there is an error. + */ + @Nullable + private TestResultData readCacheStatus() { + try (InputStream in = cacheStatus.getPath().getInputStream()) { + return TestResultData.parseFrom(in); + } catch (IOException expected) { + + } + return null; + } + + private boolean updateExecuteUnconditionallyFromTestStatus() { + if (configuration.cacheTestResults() == TriState.NO || testProperties.isExternal() + || (configuration.cacheTestResults() == TriState.AUTO + && configuration.getRunsPerTestForLabel(getOwner().getLabel()) > 1)) { + return true; + } + + // Test will not be executed unconditionally - check whether test result exists and is + // valid. If it is, method will return false and we will rely on the dependency checker + // to make a decision about test execution. + TestResultData status = readCacheStatus(); + if (status != null) { + if (!status.getCachable()) { + return true; + } + + return (configuration.cacheTestResults() == TriState.AUTO + && !status.getTestPassed()); + } + + // CacheStatus is an artifact, so if it does not exist, the dependency checker will rebuild + // it. We can't return "true" here, as it also signals to not accept cached remote results. + return false; + } + + /** + * May only be called after the dependency checked called executeUnconditionally(). + * Returns whether caching has been deemed safe by looking at the previous test run + * (for local caching). If the previous run is not present, return "true" here, as + * remote execution caching should be safe. + */ + public boolean shouldCacheResult() { + Preconditions.checkState(checkedCaching); + return !unconditionalExecution; + } + + @Override + public void actionCacheHit(Executor executor) { + checkedCaching = false; + try { + executor.getEventBus().post( + executor.getContext(TestActionContext.class).newCachedTestResult( + executor.getExecRoot(), this, readCacheStatus())); + } catch (IOException e) { + LoggingUtil.logToRemote(Level.WARNING, "Failed creating cached protocol buffer", e); + } + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + // return null here to indicate that resources would be managed manually + // during action execution. + return null; + } + + @Override + protected String getRawProgressMessage() { + return "Testing " + getTestName(); + } + + @Override + public String describeStrategy(Executor executor) { + return executor.getContext(TestActionContext.class).strategyLocality(this); + } + + /** + * Deletes <b>all</b> possible test outputs. + * + * TestRunnerAction potentially can create many more non-declared outputs - xml output, + * coverage data file and logs for failed attempts. All those outputs are uniquely + * identified by the test log base name with arbitrary prefix and extension. + */ + @Override + protected void deleteOutputs(Path execRoot) throws IOException { + super.deleteOutputs(execRoot); + + // We do not rely on globs, as it causes quadratic behavior in --runs_per_test and test + // shard count. + + // We also need to remove *.(xml|data|shard|warnings|zip) files if they are present. + execRoot.getRelative(xmlOutputPath).delete(); + execRoot.getRelative(testWarningsPath).delete(); + // Note that splitLogsPath points to a file inside the splitLogsDir so + // it's not necessary to delete it explicitly. + FileSystemUtils.deleteTree(execRoot.getRelative(splitLogsDir)); + FileSystemUtils.deleteTree(execRoot.getRelative(undeclaredOutputsDir)); + FileSystemUtils.deleteTree(execRoot.getRelative(undeclaredOutputsAnnotationsDir)); + execRoot.getRelative(testStderr).delete(); + execRoot.getRelative(testExitSafe).delete(); + if (testShard != null) { + execRoot.getRelative(testShard).delete(); + } + execRoot.getRelative(testInfrastructureFailure).delete(); + + // Coverage files use "coverage" instead of "test". + String coveragePrefix = "coverage" + namePrefix.substring(4); + + // We cannot use coverageData artifact since it may be null. Generate coverage name instead. + execRoot.getRelative(baseDir.getChild(coveragePrefix + ".dat")).delete(); + // We cannot use microcoverageData artifact since it may be null. Generate filename instead. + execRoot.getRelative(baseDir.getChild(coveragePrefix + ".micro.dat")).delete(); + + // Delete files fetched from remote execution. + execRoot.getRelative(baseDir.getChild(namePrefix + ".zip")).delete(); + deleteTestAttemptsDirMaybe(execRoot.getRelative(baseDir), namePrefix); + } + + private void deleteTestAttemptsDirMaybe(Path outputDir, String namePrefix) throws IOException { + Path testAttemptsDir = outputDir.getChild(namePrefix + "_attempts"); + if (testAttemptsDir.exists()) { + // Normally we should have used deleteTree(testAttemptsDir). However, if test output is + // in a FUSE filesystem implemented with the high-level API, there may be .fuse??????? + // entries, which prevent removing the directory. As a workaround, code below will throw + // IOException if it will fail to remove something inside testAttemptsDir, but will + // silently suppress any exceptions when deleting testAttemptsDir itself. + FileSystemUtils.deleteTreesBelow(testAttemptsDir); + try { + testAttemptsDir.delete(); + } catch (IOException e) { + // Do nothing. + } + } + } + + /** + * Gets the test name in a user-friendly format. + * Will generally include the target name and run/shard numbers, if applicable. + */ + public String getTestName() { + String suffix = getTestSuffix(); + String label = Label.print(getOwner().getLabel()); + return suffix.isEmpty() ? label : label + " " + suffix; + } + + /** + * Gets the test suffix in a user-friendly format, eg "(shard 1 of 7)". + * Will include the target name and run/shard numbers, if applicable. + */ + public String getTestSuffix() { + int totalShards = executionSettings.getTotalShards(); + // Use a 1-based index for user friendliness. + int runsPerTest = configuration.getRunsPerTestForLabel(getOwner().getLabel()); + if (totalShards > 1 && runsPerTest > 1) { + return String.format("(shard %d of %d, run %d of %d)", shardNum + 1, totalShards, + runNumber + 1, runsPerTest); + } else if (totalShards > 1) { + return String.format("(shard %d of %d)", shardNum + 1, totalShards); + } else if (runsPerTest > 1) { + return String.format("(run %d of %d)", runNumber + 1, runsPerTest); + } else { + return ""; + } + } + + public Artifact getTestLog() { + return testLog; + } + + public ResolvedPaths resolve(Path execRoot) { + return new ResolvedPaths(execRoot); + } + + public Artifact getCacheStatusArtifact() { + return cacheStatus; + } + + public PathFragment getTestWarningsPath() { + return testWarningsPath; + } + + public PathFragment getSplitLogsPath() { + return splitLogsPath; + } + + /** + * @return path to the optional zip file of undeclared test outputs. + */ + public PathFragment getUndeclaredOutputsZipPath() { + return undeclaredOutputsZipPath; + } + + /** + * @return path to the undeclared output manifest file. + */ + public PathFragment getUndeclaredOutputsManifestPath() { + return undeclaredOutputsManifestPath; + } + + /** + * @return path to the undeclared output annotations file. + */ + public PathFragment getUndeclaredOutputsAnnotationsPath() { + return undeclaredOutputsAnnotationsPath; + } + + public PathFragment getTestShard() { + return testShard; + } + + public PathFragment getExitSafeFile() { + return testExitSafe; + } + + public PathFragment getInfrastructureFailureFile() { + return testInfrastructureFailure; + } + + /** + * @return path to the optionally created XML output file created by the test. + */ + public PathFragment getXmlOutputPath() { + return xmlOutputPath; + } + + /** + * @return coverage data artifact or null if code coverage was not requested. + */ + @Nullable public Artifact getCoverageData() { + return coverageData; + } + + /** + * @return microcoverage data artifact or null if code coverage was not requested. + */ + @Nullable public Artifact getMicroCoverageData() { + return microCoverageData; + } + + public TestTargetProperties getTestProperties() { + return testProperties; + } + + public TestTargetExecutionSettings getExecutionSettings() { + return executionSettings; + } + + public boolean isSharded() { + return testShard != null; + } + + /** + * @return the shard number for this action. + * If getTotalShards() > 0, must be >= 0 and < getTotalShards(). + * Otherwise, must be 0. + */ + public int getShardNum() { + return shardNum; + } + + /** + * @return run number. + */ + public int getRunNumber() { + return runNumber; + } + + @Override + public Artifact getPrimaryOutput() { + return testLog; + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + TestActionContext context = + actionExecutionContext.getExecutor().getContext(TestActionContext.class); + try { + context.exec(this, actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException(this); + } finally { + checkedCaching = false; + } + } + + @Override + public String getMnemonic() { + return "TestRunner"; + } + + /** + * The same set of paths as the parent test action, resolved against a given exec root. + */ + public final class ResolvedPaths { + private final Path execRoot; + + ResolvedPaths(Path execRoot) { + this.execRoot = Preconditions.checkNotNull(execRoot); + } + + private Path getPath(PathFragment relativePath) { + return execRoot.getRelative(relativePath); + } + + public final Path getBaseDir() { + return getPath(baseDir); + } + + /** + * In rare cases, error messages will be printed to stderr instead of stdout. The test action is + * responsible for appending anything in the stderr file to the real test.log. + */ + public Path getTestStderr() { + return getPath(testStderr); + } + + public Path getTestWarningsPath() { + return getPath(testWarningsPath); + } + + public Path getSplitLogsPath() { + return getPath(splitLogsPath); + } + + /** + * @return path to the directory containing the split logs (raw and proto file). + */ + public Path getSplitLogsDir() { + return getPath(splitLogsDir); + } + + /** + * @return path to the optional zip file of undeclared test outputs. + */ + public Path getUndeclaredOutputsZipPath() { + return getPath(undeclaredOutputsZipPath); + } + + /** + * @return path to the directory to hold undeclared test outputs. + */ + public Path getUndeclaredOutputsDir() { + return getPath(undeclaredOutputsDir); + } + + /** + * @return path to the directory to hold undeclared output annotations parts. + */ + public Path getUndeclaredOutputsAnnotationsDir() { + return getPath(undeclaredOutputsAnnotationsDir); + } + + /** + * @return path to the undeclared output manifest file. + */ + public Path getUndeclaredOutputsManifestPath() { + return getPath(undeclaredOutputsManifestPath); + } + + /** + * @return path to the undeclared output annotations file. + */ + public Path getUndeclaredOutputsAnnotationsPath() { + return getPath(undeclaredOutputsAnnotationsPath); + } + + @Nullable + public Path getTestShard() { + return testShard == null ? null : getPath(testShard); + } + + public Path getExitSafeFile() { + return getPath(testExitSafe); + } + + public Path getInfrastructureFailureFile() { + return getPath(testInfrastructureFailure); + } + + /** + * @return path to the optionally created XML output file created by the test. + */ + public Path getXmlOutputPath() { + return getPath(xmlOutputPath); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestStrategy.java new file mode 100644 index 0000000000..4905e15293 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestStrategy.java @@ -0,0 +1,388 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.io.ByteStreams; +import com.google.common.io.Closeables; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.analysis.config.BinTools; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.exec.SymlinkTreeHelper; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.runtime.BlazeServerStartupOptions; +import com.google.devtools.build.lib.util.io.FileWatcher; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.FileStatus; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.test.TestStatus.TestCase; +import com.google.devtools.common.options.Converters.RangeConverter; +import com.google.devtools.common.options.EnumConverter; +import com.google.devtools.common.options.OptionsClassProvider; +import com.google.devtools.common.options.OptionsParsingException; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * A strategy for executing a {@link TestRunnerAction}. + */ +public abstract class TestStrategy implements TestActionContext { + /** + * Converter for the --flaky_test_attempts option. + */ + public static class TestAttemptsConverter extends RangeConverter { + public TestAttemptsConverter() { + super(1, 10); + } + + @Override + public Integer convert(String input) throws OptionsParsingException { + if ("default".equals(input)) { + return -1; + } else { + return super.convert(input); + } + } + + @Override + public String getTypeDescription() { + return super.getTypeDescription() + " or the string \"default\""; + } + } + + public enum TestOutputFormat { + SUMMARY, // Provide summary output only. + ERRORS, // Print output from failed tests to the stderr after the test failure. + ALL, // Print output from all tests to the stderr after the test completion. + STREAMED; // Stream output for each test. + + /** + * Converts to {@link TestOutputFormat}. + */ + public static class Converter extends EnumConverter<TestOutputFormat> { + public Converter() { + super(TestOutputFormat.class, "test output"); + } + } + } + + public enum TestSummaryFormat { + SHORT, // Print information only about tests. + TERSE, // Like "SHORT", but even shorter: Do not print PASSED tests. + DETAILED, // Print information only about failed test cases. + NONE; // Do not print summary. + + /** + * Converts to {@link TestSummaryFormat}. + */ + public static class Converter extends EnumConverter<TestSummaryFormat> { + public Converter() { + super(TestSummaryFormat.class, "test summary"); + } + } + } + + public static final PathFragment TEST_TMP_ROOT = new PathFragment("_tmp"); + + // Used for selecting subset of testcase / testmethods. + private static final String TEST_BRIDGE_TEST_FILTER_ENV = "TESTBRIDGE_TEST_ONLY"; + + private final boolean statusServerRunning; + protected final ExecutionOptions executionOptions; + protected final BinTools binTools; + + public TestStrategy(OptionsClassProvider requestOptionsProvider, + OptionsClassProvider startupOptionsProvider, BinTools binTools) { + this.executionOptions = requestOptionsProvider.getOptions(ExecutionOptions.class); + this.binTools = binTools; + BlazeServerStartupOptions startupOptions = + startupOptionsProvider.getOptions(BlazeServerStartupOptions.class); + statusServerRunning = startupOptions != null && startupOptions.useWebStatusServer > 0; + } + + @Override + public abstract void exec(TestRunnerAction action, ActionExecutionContext actionExecutionContext) + throws ExecException, InterruptedException; + + @Override + public abstract String strategyLocality(TestRunnerAction action); + + /** + * Callback for determining the strategy locality. + * + * @param action the test action + * @param localRun whether to run it locally + */ + protected String strategyLocality(TestRunnerAction action, boolean localRun) { + return strategyLocality(action); + } + + /** + * Returns mutable map of default testing shell environment. By itself it is incomplete and is + * modified further by the specific test strategy implementations (mostly due to the fact that + * environments used locally and remotely are different). + */ + protected Map<String, String> getDefaultTestEnvironment(TestRunnerAction action) { + Map<String, String> env = new HashMap<>(); + + env.putAll(action.getConfiguration().getDefaultShellEnvironment()); + env.remove("LANG"); + env.put("TZ", "UTC"); + env.put("TEST_SIZE", action.getTestProperties().getSize().toString()); + env.put("TEST_TIMEOUT", Integer.toString(getTimeout(action))); + + if (action.isSharded()) { + env.put("TEST_SHARD_INDEX", Integer.toString(action.getShardNum())); + env.put("TEST_TOTAL_SHARDS", + Integer.toString(action.getExecutionSettings().getTotalShards())); + } + + // When we run test multiple times, set different TEST_RANDOM_SEED values for each run. + if (action.getConfiguration().getRunsPerTestForLabel(action.getOwner().getLabel()) > 1) { + env.put("TEST_RANDOM_SEED", Integer.toString(action.getRunNumber() + 1)); + } + + String testFilter = action.getExecutionSettings().getTestFilter(); + if (testFilter != null) { + env.put(TEST_BRIDGE_TEST_FILTER_ENV, testFilter); + } + + return env; + } + + /** + * Returns the number of attempts specific test action can be retried. + * + * <p>For rules with "flaky = 1" attribute, this method will return 3 unless --flaky_test_attempts + * option is given and specifies another value. + */ + @VisibleForTesting /* protected */ + public int getTestAttempts(TestRunnerAction action) { + if (executionOptions.testAttempts == -1) { + return action.getTestProperties().isFlaky() ? 3 : 1; + } else { + return executionOptions.testAttempts; + } + } + + /** + * Returns timeout value in seconds that should be used for the given test action. We always use + * the "categorical timeouts" which are based on the --test_timeout flag. A rule picks its timeout + * but ends up with the same effective value as all other rules in that bucket. + */ + protected final int getTimeout(TestRunnerAction testAction) { + return executionOptions.testTimeout.get(testAction.getTestProperties().getTimeout()); + } + + /** + * Returns a subset of the environment from the current shell. + * + * <p>Warning: Since these variables are not part of the configuration's fingerprint, they + * MUST NOT be used by any rule or action in such a way as to affect the semantics of that + * build step. + */ + public Map<String, String> getAdmissibleShellEnvironment(BuildConfiguration config, + Iterable<String> variables) { + return getMapping(variables, config.getClientEnv()); + } + + /* + * Finalize test run: persist the result, and post on the event bus. + */ + protected void postTestResult(Executor executor, TestResult result) throws IOException { + result.getTestAction().saveCacheStatus(result.getData()); + executor.getEventBus().post(result); + } + + /** + * Parse a test result XML file into a {@link TestCase}. + */ + @Nullable + protected TestCase parseTestResult(Path resultFile) { + /* xml files. We avoid parsing it unnecessarily, since test results can potentially consume + a large amount of memory. */ + if (executionOptions.testSummary != TestSummaryFormat.DETAILED && !statusServerRunning) { + return null; + } + + try (InputStream fileStream = resultFile.getInputStream()) { + return new TestXmlOutputParser().parseXmlIntoTestResult(fileStream); + } catch (IOException | TestXmlOutputParserException e) { + return null; + } + } + + /** + * For an given environment, returns a subset containing all variables in the given list if they + * are defined in the given environment. + */ + @VisibleForTesting + public static Map<String, String> getMapping(Iterable<String> variables, + Map<String, String> environment) { + Map<String, String> result = new HashMap<>(); + for (String var : variables) { + if (environment.containsKey(var)) { + result.put(var, environment.get(var)); + } + } + return result; + } + + /** + * Returns the runfiles directory associated with the test executable, + * creating/updating it if necessary and --build_runfile_links is specified. + */ + protected static Path getLocalRunfilesDirectory(TestRunnerAction testAction, + ActionExecutionContext actionExecutionContext, BinTools binTools) throws ExecException, + InterruptedException { + TestTargetExecutionSettings execSettings = testAction.getExecutionSettings(); + + // --nobuild_runfile_links disables runfiles generation only for C++ rules. + // In that case, getManifest returns the .runfiles_manifest (input) file, + // not the MANIFEST output file of the build-runfiles action. So the + // extension ".runfiles_manifest" indicates no runfiles tree. + if (!execSettings.getManifest().equals(execSettings.getInputManifest())) { + return execSettings.getManifest().getPath().getParentDirectory(); + } + + // We might need to build runfiles tree now, since it was not created yet + // local testing is needed. + Path program = execSettings.getExecutable().getPath(); + Path runfilesDir = program.getParentDirectory().getChild(program.getBaseName() + ".runfiles"); + + // Synchronize runfiles tree generation on the runfiles manifest artifact. + // This is necessary, because we might end up with multiple test runner actions + // trying to generate same runfiles tree in case of --runs_per_test > 1 or + // local test sharding. + long startTime = Profiler.nanoTimeMaybe(); + synchronized (execSettings.getManifest()) { + Profiler.instance().logSimpleTask(startTime, ProfilerTask.WAIT, testAction); + updateLocalRunfilesDirectory(testAction, runfilesDir, actionExecutionContext, binTools); + } + + return runfilesDir; + } + + /** + * Ensure the runfiles tree exists and is consistent with the TestAction's manifest + * ($0.runfiles_manifest), bringing it into consistency if not. The contents of the output file + * $0.runfiles/MANIFEST, if it exists, are used a proxy for the set of existing symlinks, to avoid + * the need for recursion. + */ + private static void updateLocalRunfilesDirectory(TestRunnerAction testAction, Path runfilesDir, + ActionExecutionContext actionExecutionContext, BinTools binTools) throws ExecException, + InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + + TestTargetExecutionSettings execSettings = testAction.getExecutionSettings(); + try { + if (Arrays.equals(runfilesDir.getRelative("MANIFEST").getMD5Digest(), + execSettings.getManifest().getPath().getMD5Digest())) { + return; + } + } catch (IOException e1) { + // Ignore it - we will just try to create runfiles directory. + } + + executor.getEventHandler().handle(Event.progress( + "Building runfiles directory for '" + execSettings.getExecutable().prettyPrint() + "'.")); + + new SymlinkTreeHelper(execSettings.getManifest().getExecPath(), + runfilesDir.relativeTo(executor.getExecRoot()), /* filesetTree= */ false) + .createSymlinks(testAction, actionExecutionContext, binTools); + + executor.getEventHandler().handle(Event.progress(testAction.getProgressMessage())); + } + + /** + * In rare cases, we might write something to stderr. Append it to the real test.log. + */ + protected static void appendStderr(Path stdOut, Path stdErr) throws IOException { + FileStatus stat = stdErr.statNullable(); + OutputStream out = null; + InputStream in = null; + if (stat != null) { + try { + if (stat.getSize() > 0) { + if (stdOut.exists()) { + stdOut.setWritable(true); + } + out = stdOut.getOutputStream(true); + in = stdErr.getInputStream(); + ByteStreams.copy(in, out); + } + } finally { + Closeables.close(out, true); + Closeables.close(in, true); + stdErr.delete(); + } + } + } + + /** + * Implements the --test_output=streamed option. + */ + protected static class StreamedTestOutput implements Closeable { + private final TestLogHelper.FilterTestHeaderOutputStream headerFilter; + private final FileWatcher watcher; + private final Path testLogPath; + private final OutErr outErr; + + public StreamedTestOutput(OutErr outErr, Path testLogPath) throws IOException { + this.testLogPath = testLogPath; + this.outErr = outErr; + this.headerFilter = TestLogHelper.getHeaderFilteringOutputStream(outErr.getOutputStream()); + this.watcher = new FileWatcher(testLogPath, OutErr.create(headerFilter, headerFilter), false); + watcher.start(); + } + + @Override + public void close() throws IOException { + watcher.stopPumping(); + try { + // The watcher thread might leak if the following call is interrupted. + // This is a relatively minor issue since the worst it could do is + // write one additional line from the test.log to the console later on + // in the build. + watcher.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (!headerFilter.foundHeader()) { + InputStream input = testLogPath.getInputStream(); + try { + ByteStreams.copy(input, outErr.getOutputStream()); + } finally { + input.close(); + } + } + } + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestSuite.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestSuite.java new file mode 100644 index 0000000000..ef795aa617 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestSuite.java @@ -0,0 +1,99 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.test; + +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.packages.TestTargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.util.Pair; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Implementation for the "test_suite" rule. + */ +public class TestSuite implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + checkTestsAndSuites(ruleContext, "tests"); + checkTestsAndSuites(ruleContext, "suites"); + if (ruleContext.hasErrors()) { + return null; + } + + // + // CAUTION! Keep this logic consistent with lib.query2.TestsExpression! + // + + List<String> tagsAttribute = new ArrayList<>( + ruleContext.attributes().get("tags", Type.STRING_LIST)); + tagsAttribute.remove("manual"); + Pair<Collection<String>, Collection<String>> requiredExcluded = + TestTargetUtils.sortTagsBySense(tagsAttribute); + + List<TransitiveInfoCollection> directTestsAndSuitesBuilder = new ArrayList<>(); + + // The set of implicit tests is determined in + // {@link com.google.devtools.build.lib.packages.Package}. + // Manual tests are already filtered out there. That is what $implicit_tests is about. + for (TransitiveInfoCollection dep : + Iterables.concat( + ruleContext.getPrerequisites("tests", Mode.TARGET), + ruleContext.getPrerequisites("suites", Mode.TARGET), + ruleContext.getPrerequisites("$implicit_tests", Mode.TARGET))) { + if (dep.getProvider(TestProvider.class) != null) { + List<String> tags = dep.getProvider(TestProvider.class).getTestTags(); + if (!TestTargetUtils.testMatchesFilters( + tags, requiredExcluded.first, requiredExcluded.second, true)) { + // This test does not match our filter. Ignore it. + continue; + } + } + directTestsAndSuitesBuilder.add(dep); + } + + Runfiles runfiles = new Runfiles.Builder() + .addTargets(directTestsAndSuitesBuilder, RunfilesProvider.DATA_RUNFILES) + .build(); + + return new RuleConfiguredTargetBuilder(ruleContext) + .add(RunfilesProvider.class, + RunfilesProvider.withData(Runfiles.EMPTY, runfiles)) + .add(TransitiveTestsProvider.class, new TransitiveTestsProvider()) + .build(); + } + + private void checkTestsAndSuites(RuleContext ruleContext, String attributeName) { + for (TransitiveInfoCollection dep : ruleContext.getPrerequisites(attributeName, Mode.TARGET)) { + // TODO(bazel-team): Maybe convert the TransitiveTestsProvider into an inner interface. + TransitiveTestsProvider provider = dep.getProvider(TransitiveTestsProvider.class); + TestProvider testProvider = dep.getProvider(TestProvider.class); + if (provider == null && testProvider == null) { + ruleContext.attributeError(attributeName, + "expecting a test or a test_suite rule but '" + dep.getLabel() + "' is not one"); + } + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetExecutionSettings.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetExecutionSettings.java new file mode 100644 index 0000000000..20ad8af5ec --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetExecutionSettings.java @@ -0,0 +1,133 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.RunUnder; +import com.google.devtools.build.lib.packages.TargetUtils; + +import java.util.List; + +/** + * Container for common test execution settings shared by all + * all TestRunnerAction instances for the given test target. + */ +public final class TestTargetExecutionSettings { + + private final List<String> testArguments; + private final String testFilter; + private final int totalShards; + private final RunUnder runUnder; + private final Artifact runUnderExecutable; + private final Artifact executable; + private final Artifact runfilesManifest; + private final Artifact runfilesInputManifest; + private final Artifact instrumentedFileManifest; + + TestTargetExecutionSettings(RuleContext ruleContext, RunfilesSupport runfiles, + Artifact executable, Artifact instrumentedFileManifest, int shards) { + Preconditions.checkArgument(TargetUtils.isTestRule(ruleContext.getRule())); + Preconditions.checkArgument(shards >= 0); + BuildConfiguration config = ruleContext.getConfiguration(); + + List<String> targetArgs = runfiles.getArgs(); + testArguments = targetArgs.isEmpty() + ? config.getTestArguments() + : ImmutableList.copyOf(Iterables.concat(targetArgs, config.getTestArguments())); + + totalShards = shards; + runUnder = config.getRunUnder(); + runUnderExecutable = getRunUnderExecutable(ruleContext); + + this.testFilter = config.getTestFilter(); + this.executable = executable; + this.runfilesManifest = runfiles.getRunfilesManifest(); + this.runfilesInputManifest = runfiles.getRunfilesInputManifest(); + this.instrumentedFileManifest = instrumentedFileManifest; + } + + private static Artifact getRunUnderExecutable(RuleContext ruleContext) { + TransitiveInfoCollection runUnderTarget = ruleContext + .getPrerequisite(":run_under", Mode.DATA); + return runUnderTarget == null + ? null + : runUnderTarget.getProvider(FilesToRunProvider.class).getExecutable(); + } + + public List<String> getArgs() { + return testArguments; + } + + public String getTestFilter() { + return testFilter; + } + + public int getTotalShards() { + return totalShards; + } + + public RunUnder getRunUnder() { + return runUnder; + } + + public Artifact getRunUnderExecutable() { + return runUnderExecutable; + } + + public Artifact getExecutable() { + return executable; + } + + /** + * Returns the runfiles manifest for this test. + * + * <p>This returns either the input manifest outside of the runfiles tree, + * if blaze is run with --nobuild_runfile_links or the manifest inside the + * runfiles tree, if blaze is run with --build_runfile_links. + * + * @see com.google.devtools.build.lib.analysis.RunfilesSupport#getRunfilesManifest() + */ + public Artifact getManifest() { + return runfilesManifest; + } + + /** + * Returns the input runfiles manifest for this test. + * + * <p>This always returns the input manifest outside of the runfiles tree. + * + * @see com.google.devtools.build.lib.analysis.RunfilesSupport#getRunfilesInputManifest() + */ + public Artifact getInputManifest() { + return runfilesInputManifest; + } + + /** + * Returns instrumented file manifest or null if code coverage is not + * collected. + */ + public Artifact getInstrumentedFileManifest() { + return instrumentedFileManifest; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java new file mode 100644 index 0000000000..8cf26b8dc1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java @@ -0,0 +1,131 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.TestSize; +import com.google.devtools.build.lib.packages.TestTimeout; +import com.google.devtools.build.lib.packages.Type; + +import java.util.List; +import java.util.Map; + +/** + * Container for test target properties available to the + * TestRunnerAction instance. + */ +public class TestTargetProperties { + + /** + * Resources used by local tests of various sizes. + */ + private static final ResourceSet SMALL_RESOURCES = new ResourceSet(20, 0.9, 0.00); + private static final ResourceSet MEDIUM_RESOURCES = new ResourceSet(100, 0.9, 0.1); + private static final ResourceSet LARGE_RESOURCES = new ResourceSet(300, 0.8, 0.1); + private static final ResourceSet ENORMOUS_RESOURCES = new ResourceSet(800, 0.7, 0.4); + + private static ResourceSet getResourceSetFromSize(TestSize size) { + switch (size) { + case SMALL: return SMALL_RESOURCES; + case MEDIUM: return MEDIUM_RESOURCES; + case LARGE: return LARGE_RESOURCES; + default: return ENORMOUS_RESOURCES; + } + } + + private final TestSize size; + private final TestTimeout timeout; + private final List<String> tags; + private final boolean isLocal; + private final boolean isFlaky; + private final boolean isExternal; + private final String language; + private final ImmutableMap<String, String> executionInfo; + + /** + * Creates test target properties instance. Constructor expects that it + * will be called only for test configured targets. + */ + TestTargetProperties(RuleContext ruleContext, + ExecutionInfoProvider executionRequirements) { + Rule rule = ruleContext.getRule(); + + Preconditions.checkState(TargetUtils.isTestRule(rule)); + size = TestSize.getTestSize(rule); + timeout = TestTimeout.getTestTimeout(rule); + tags = ruleContext.attributes().get("tags", Type.STRING_LIST); + isLocal = TargetUtils.isLocalTestRule(rule) || TargetUtils.isExclusiveTestRule(rule); + + // We need to use method on ruleConfiguredTarget to perform validation. + isFlaky = ruleContext.attributes().get("flaky", Type.BOOLEAN); + isExternal = TargetUtils.isExternalTestRule(rule); + + Map<String, String> executionInfo = Maps.newLinkedHashMap(); + executionInfo.putAll(TargetUtils.getExecutionInfo(rule)); + if (executionRequirements != null) { + // This will overwrite whatever TargetUtils put there, which might be confusing. + executionInfo.putAll(executionRequirements.getExecutionInfo()); + } + this.executionInfo = ImmutableMap.copyOf(executionInfo); + + language = TargetUtils.getRuleLanguage(rule); + } + + public TestSize getSize() { + return size; + } + + public TestTimeout getTimeout() { + return timeout; + } + + public List<String> getTags() { + return tags; + } + + public boolean isLocal() { + return isLocal; + } + + public boolean isFlaky() { + return isFlaky; + } + + public boolean isExternal() { + return isExternal; + } + + public ResourceSet getLocalResourceUsage() { + return TestTargetProperties.getResourceSetFromSize(size); + } + + /** + * Returns a map of execution info. See {@link Spawn#getExecutionInfo}. + */ + public ImmutableMap<String, String> getExecutionInfo() { + return executionInfo; + } + + public String getLanguage() { + return language; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParser.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParser.java new file mode 100644 index 0000000000..8d660ec467 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParser.java @@ -0,0 +1,345 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.view.test.TestStatus.TestCase; +import com.google.devtools.build.lib.view.test.TestStatus.TestCase.Type; +import com.google.protobuf.UninitializedMessageException; + +import java.io.InputStream; +import java.util.Collection; + +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +/** + * Parses a test.xml generated by jUnit or any testing framework + * into a protocol buffer. The schema of the test.xml is a bit hazy, so there is + * some guesswork involved. + */ +class TestXmlOutputParser { + // jUnit can use either "testsuites" or "testsuite". + private static final Collection<String> TOPLEVEL_ELEMENT_NAMES = + ImmutableSet.of("testsuites", "testsuite"); + + public TestCase parseXmlIntoTestResult(InputStream xmlStream) + throws TestXmlOutputParserException { + return parseXmlToTree(xmlStream); + } + + /** + * Parses the a test result XML file into the corresponding protocol buffer. + * @param xmlStream the XML data stream + * @return the protocol buffer with the parsed data, or null if there was + * an error while parsing the file. + * + * @throws TestXmlOutputParserException when the XML file cannot be parsed + */ + private TestCase parseXmlToTree(InputStream xmlStream) + throws TestXmlOutputParserException { + XMLStreamReader parser = null; + + try { + parser = XMLInputFactory.newInstance().createXMLStreamReader(xmlStream); + + while (true) { + int event = parser.next(); + if (event == XMLStreamConstants.END_DOCUMENT) { + return null; + } + + // First find the topmost node. + if (event == XMLStreamConstants.START_ELEMENT) { + String elementName = parser.getLocalName(); + if (TOPLEVEL_ELEMENT_NAMES.contains(elementName)) { + TestCase result = parseTestSuite(parser, elementName); + return result; + } + } + } + } catch (XMLStreamException e) { + throw new TestXmlOutputParserException(e); + } catch (NumberFormatException e) { + // The parser is definitely != null here. + throw new TestXmlOutputParserException( + "Number could not be parsed at " + + parser.getLocation().getLineNumber() + ":" + + parser.getLocation().getColumnNumber(), + e); + } catch (UninitializedMessageException e) { + // This happens when the XML does not contain a field that is required + // in the protocol buffer + throw new TestXmlOutputParserException(e); + } catch (RuntimeException e) { + + // Seems like that an XNIException can leak through, even though it is not + // specified anywhere. + // + // It's a bad idea to refer to XNIException directly because the Xerces + // documentation says that it may not be available here soon (and it + // results in a compile-time warning anyway), so we do it the roundabout + // way: check if the class name has something to do with Xerces, and if + // so, wrap it in our own exception type, otherwise, let the stack + // unwinding continue. + String name = e.getClass().getCanonicalName(); + if (name != null && name.contains("org.apache.xerces")) { + throw new TestXmlOutputParserException(e); + } else { + throw e; + } + } finally { + if (parser != null) { + try { + parser.close(); + } catch (XMLStreamException e) { + + // Ignore errors during closure so that we do not interfere with an + // already propagating exception. + } + } + } + } + + /** + * Creates an exception suitable to be thrown when and a bad end tag appears. + * The exception could also be thrown from here but that would result in an + * extra stack frame, whereas this way, the topmost frame shows the location + * where the error occurred. + */ + private TestXmlOutputParserException createBadElementException( + String expected, XMLStreamReader parser) { + return new TestXmlOutputParserException("Expected end of XML element '" + + expected + "' , but got '" + parser.getLocalName() + "' at " + + parser.getLocation().getLineNumber() + ":" + + parser.getLocation().getColumnNumber()); + } + + /** + * Parses a 'testsuite' element. + * + * @throws TestXmlOutputParserException if the XML document is malformed + * @throws XMLStreamException if there was an error processing the XML + * @throws NumberFormatException if one of the numeric fields does not contain + * a valid number + */ + private TestCase parseTestSuite(XMLStreamReader parser, String elementName) + throws XMLStreamException, TestXmlOutputParserException { + TestCase.Builder builder = TestCase.newBuilder(); + builder.setType(Type.TEST_SUITE); + for (int i = 0; i < parser.getAttributeCount(); i++) { + String name = parser.getAttributeLocalName(i).intern(); + String value = parser.getAttributeValue(i); + + if (name.equals("name")) { + builder.setName(value); + } else if (name.equals("time")) { + builder.setRunDurationMillis(parseTime(value)); + } + } + + parseContainedElements(parser, elementName, builder); + return builder.build(); + } + + /** + * Parses a time in test.xml format. + * + * @throws NumberFormatException if the time is malformed (i.e. is neither an + * integer nor a decimal fraction with '.' as the fraction separator) + */ + private long parseTime(String string) { + + // This is ugly. For Historical Reasons, we have to check whether the number + // contains a decimal point or not. If it does, the number is expressed in + // milliseconds, otherwise, in seconds. + if (string.contains(".")) { + return Math.round(Float.parseFloat(string) * 1000); + } else { + return Long.parseLong(string); + } + } + + /** + * Parses a 'decorator' element. + * + * @throws TestXmlOutputParserException if the XML document is malformed + * @throws XMLStreamException if there was an error processing the XML + * @throws NumberFormatException if one of the numeric fields does not contain + * a valid number + */ + private TestCase parseTestDecorator(XMLStreamReader parser) + throws XMLStreamException, TestXmlOutputParserException { + TestCase.Builder builder = TestCase.newBuilder(); + builder.setType(Type.TEST_DECORATOR); + for (int i = 0; i < parser.getAttributeCount(); i++) { + String name = parser.getAttributeLocalName(i); + String value = parser.getAttributeValue(i); + + builder.setName(name); + if (name.equals("classname")) { + builder.setClassName(value); + } else if (name.equals("time")) { + builder.setRunDurationMillis(parseTime(value)); + } + } + + parseContainedElements(parser, "testdecorator", builder); + return builder.build(); + } + + /** + * Parses child elements of the specified tag. Strictly speaking, not every + * element can be a child of every other, but the HierarchicalTestResult can + * handle that, and (in this case) it does not hurt to be a bit more flexible + * than necessary. + * + * @throws TestXmlOutputParserException if the XML document is malformed + * @throws XMLStreamException if there was an error processing the XML + * @throws NumberFormatException if one of the numeric fields does not contain + * a valid number + */ + private void parseContainedElements( + XMLStreamReader parser, String elementName, TestCase.Builder builder) + throws XMLStreamException, TestXmlOutputParserException { + int failures = 0; + int errors = 0; + + while (true) { + int event = parser.next(); + switch (event) { + case XMLStreamConstants.START_ELEMENT: + String childElementName = parser.getLocalName().intern(); + + // We are not parsing four elements here: system-out, system-err, + // failure and error. They potentially contain useful information, but + // they can be too big to fit in the memory. We add failure and error + // elements to the output without a message, so that there is a + // difference between passed and failed test cases. + if (childElementName.equals("testsuite")) { + builder.addChild(parseTestSuite(parser, childElementName)); + } else if (childElementName.equals("testcase")) { + builder.addChild(parseTestCase(parser)); + } else if (childElementName.equals("failure")) { + failures += 1; + skipCompleteElement(parser); + } else if (childElementName.equals("error")) { + errors += 1; + skipCompleteElement(parser); + } else if (childElementName.equals("testdecorator")) { + builder.addChild(parseTestDecorator(parser)); + } else { + + // Unknown element encountered. Since the schema of the input file + // is a bit hazy, just skip it and go merrily on our way. Ignorance + // is bliss. + skipCompleteElement(parser); + } + break; + + case XMLStreamConstants.END_ELEMENT: + // Propagate errors/failures from children up to the current case + for (int i = 0; i < builder.getChildCount(); i += 1) { + if (builder.getChild(i).getStatus() == TestCase.Status.ERROR) { + errors += 1; + } + if (builder.getChild(i).getStatus() == TestCase.Status.FAILED) { + failures += 1; + } + } + + if (errors > 0) { + builder.setStatus(TestCase.Status.ERROR); + } else if (failures > 0) { + builder.setStatus(TestCase.Status.FAILED); + } else { + builder.setStatus(TestCase.Status.PASSED); + } + // This is the end tag of the element we are supposed to parse. + // Hooray, tell our superiors that our mission is complete. + if (!parser.getLocalName().equals(elementName)) { + throw createBadElementException(elementName, parser); + } + return; + } + } + } + + + /** + * Parses a 'testcase' element. + * + * @throws TestXmlOutputParserException if the XML document is malformed + * @throws XMLStreamException if there was an error processing the XML + * @throws NumberFormatException if the time field does not contain a valid + * number + */ + private TestCase parseTestCase(XMLStreamReader parser) + throws XMLStreamException, TestXmlOutputParserException { + TestCase.Builder builder = TestCase.newBuilder(); + builder.setType(Type.TEST_CASE); + for (int i = 0; i < parser.getAttributeCount(); i++) { + String name = parser.getAttributeLocalName(i).intern(); + String value = parser.getAttributeValue(i); + + if (name.equals("name")) { + builder.setName(value); + } else if (name.equals("classname")) { + builder.setClassName(value); + } else if (name.equals("time")) { + builder.setRunDurationMillis(parseTime(value)); + } else if (name.equals("result")) { + builder.setResult(value); + } else if (name.equals("status")) { + if (value.equals("notrun")) { + builder.setRun(false); + } else if (value.equals("run")) { + builder.setRun(true); + } + } + } + + parseContainedElements(parser, "testcase", builder); + return builder.build(); + } + + /** + * Skips over a complete XML element on the input. + * Precondition: the cursor is at a START_ELEMENT. + * Postcondition: the cursor is at an END_ELEMENT. + * + * @throws XMLStreamException if the XML is malformed + */ + private void skipCompleteElement(XMLStreamReader parser) throws XMLStreamException { + int depth = 1; + while (true) { + int event = parser.next(); + + switch (event) { + case XMLStreamConstants.START_ELEMENT: + depth++; + break; + + case XMLStreamConstants.END_ELEMENT: + if (--depth == 0) { + return; + } + break; + } + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParserException.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParserException.java new file mode 100644 index 0000000000..c27ca9d305 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParserException.java @@ -0,0 +1,33 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +/** + * This exception gets thrown if there was a problem with parsing a test.xml + * file. + */ +class TestXmlOutputParserException extends Exception { + public TestXmlOutputParserException(String message, Throwable cause) { + super(message, cause); + } + + public TestXmlOutputParserException(Throwable cause) { + super(cause); + } + + public TestXmlOutputParserException(String message) { + super(message); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TransitiveTestsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/test/TransitiveTestsProvider.java new file mode 100644 index 0000000000..c46b2a78dc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TransitiveTestsProvider.java @@ -0,0 +1,25 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Marker transitive info provider for test_suite rules to recognize one another. + */ +@Immutable +public final class TransitiveTestsProvider implements TransitiveInfoProvider { +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/workspace/Bind.java b/src/main/java/com/google/devtools/build/lib/rules/workspace/Bind.java new file mode 100644 index 0000000000..49f829a1c2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/workspace/Bind.java @@ -0,0 +1,125 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.workspace; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.UnmodifiableIterator; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.syntax.ClassObject; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkNestedSet; + +/** + * Implementation for the bind rule. + */ +public class Bind implements RuleConfiguredTargetFactory { + + /** + * This configured target pretends to be whatever type of target "actual" is, returning its + * transitive info providers and target, but returning the label for the //external target. + */ + private static class BindConfiguredTarget implements ConfiguredTarget, ClassObject { + + private Label label; + private ConfiguredTarget configuredTarget; + private BuildConfiguration config; + + BindConfiguredTarget(RuleContext ruleContext) { + label = ruleContext.getRule().getLabel(); + config = ruleContext.getConfiguration(); + // TODO(bazel-team): we should special case ConfiguredTargetFactory.createConfiguredTarget, + // not cast down here. + configuredTarget = (ConfiguredTarget) ruleContext.getPrerequisite("actual", Mode.TARGET); + } + + @Override + public <P extends TransitiveInfoProvider> P getProvider(Class<P> provider) { + return configuredTarget.getProvider(provider); + } + + @Override + public Label getLabel() { + return label; + } + + @Override + public Object get(String providerKey) { + return configuredTarget.get(providerKey); + } + + @Override + public UnmodifiableIterator<TransitiveInfoProvider> iterator() { + return configuredTarget.iterator(); + } + + @Override + public Target getTarget() { + return configuredTarget.getTarget(); + } + + @Override + public BuildConfiguration getConfiguration() { + return config; + } + + /* ClassObject methods */ + + @Override + public Object getValue(String name) { + if (name.equals("label")) { + return getLabel(); + } else if (name.equals("files")) { + // A shortcut for files to build in Skylark. FileConfiguredTarget and RunleConfiguredTarget + // always has FileProvider and Error- and PackageGroupConfiguredTarget-s shouldn't be + // accessible in Skylark. + return SkylarkNestedSet.of( + Artifact.class, getProvider(FileProvider.class).getFilesToBuild()); + } + return configuredTarget.get(name); + } + + @SuppressWarnings("cast") + @Override + public ImmutableCollection<String> getKeys() { + return new ImmutableList.Builder<String>() + .add("label", "files") + .addAll(configuredTarget.getProvider(RuleConfiguredTarget.SkylarkProviders.class) + .getKeys()) + .build(); + } + + @Override + public String errorMessage(String name) { + // Use the default error message. + return null; + } + } + + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + return new BindConfiguredTarget(ruleContext); + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/workspace/BindRule.java b/src/main/java/com/google/devtools/build/lib/rules/workspace/BindRule.java new file mode 100644 index 0000000000..c3f2dd2a35 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/workspace/BindRule.java @@ -0,0 +1,126 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.workspace; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses.BaseRule; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; + +/** + * Binds an existing target to a target in the virtual //external package. + */ +@BlazeRule(name = "bind", + type = RuleClassType.WORKSPACE, + ancestors = {BaseRule.class}, + factoryClass = Bind.class) +public final class BindRule implements RuleDefinition { + + @Override + public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE(bind).ATTRIBUTE(actual) --> + The target to be aliased. + + <p>This target must exist, but can be any type of rule (including bind).</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("actual", LABEL).allowedFileTypes()) + .setWorkspaceOnly() + .build(); + } +} +/*<!-- #BLAZE_RULE (NAME = bind, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] --> + +${ATTRIBUTE_SIGNATURE} + +<p>Gives a target an alias in the <code>//external</code> package.</p> + +${ATTRIBUTE_DEFINITION} + +<p>The <code>//external</code> package is not a "normal" package: there is no external/ directory, + so it can be thought of as a "virtual package" that contains all bound targets.</p> + +<h4 id="bind_examples">Examples</h4> + +<p>To give a target an alias, bind it in the <i>WORKSPACE</i> file. For example, suppose there is + a <code>java_library</code> target called <code>//third_party/javacc-v2</code>. This could be + aliased by adding the following to the <i>WORKSPACE</i> file:</p> + +<pre class="code"> +bind( + name = "javacc-latest", + actual = "//third_party/javacc-v2", +) +</pre> + +<p>Now targets can depend on <code>//external:javacc-latest</code> instead of + <code>//third_party/javacc-v2</code>. If javacc-v3 is released, the binding can be updated and + all of the BUILD files depending on <code>//external:javacc-latest</code> will now depend on + javacc-v3 without needing to be edited.</p> + +<p>Bind can also be used to refer to external repositories' targets. For example, if there is a + remote repository named <code>@my-ssl</code> imported in the WORKSPACE file. If the + <code>@my-ssl</code> repository has a cc_library target <code>//src:openssl-lib</code>, you + could make this target accessible for your program to depend on by using <code>bind</code>:</p> + +<pre class="code"> +bind( + name = "openssl", + actual = "@my-ssl//src:openssl-lib", +) +</pre> + +<p>BUILD files cannot use labels that include a repository name + ("@repository-name//package-name:target-name"), so the only way to depend on a target from + another repository is to <code>bind</code> it in the WORKSPACE file and then refer to it by its + aliased name in <code>//external</code> from a BUILD file.</p> + +<p>For example, in a BUILD file, the bound target could be used as follows:</p> + +<pre class="code"> +cc_library( + name = "sign-in", + srcs = ["sign_in.cc"], + hdrs = ["sign_in.h"], + deps = ["//external:openssl"], +) +</pre> + +<p>Within <code>sign_in.cc</code> and <code>sign_in.h</code>, the header files exposed by + <code>//external:openssl</code> can be referred to by their path relative to their repository + root. For example, if the rule definition for <code>@my-ssl//src:openssl-lib</code> looks like + this:</p> + +<pre class="code"> +cc_library( + name = "openssl-lib", + srcs = ["openssl.cc"], + hdrs = ["openssl.h"], +) +</pre> + +<p>Then <code>sign_in.cc</code>'s first lines might look like this:</p> + +<pre class="code"> +#include "sign_in.h" +#include "src/openssl.h" +</pre> + +<!-- #END_BLAZE_RULE -->*/ |