diff options
25 files changed, 739 insertions, 25 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD index 25fab2153e..124d3c58fb 100644 --- a/src/main/java/com/google/devtools/build/lib/BUILD +++ b/src/main/java/com/google/devtools/build/lib/BUILD @@ -570,7 +570,7 @@ java_library( name = "bazel-repository", srcs = glob( [ - "bazel/repository/*.java", + "bazel/repository/**/*.java", "bazel/rules/workspace/*.java", ], exclude = ["bazel/repository/MavenConnector.java"], @@ -596,6 +596,7 @@ java_library( "//src/java_tools/singlejar/java/com/google/devtools/build/zip", "//src/main/java/com/google/devtools/build/lib:build-base", "//src/main/java/com/google/devtools/build/lib:packages-internal", + "//src/main/java/com/google/devtools/build/lib:skylarkinterface", "//src/main/java/com/google/devtools/build/lib:vfs", "//src/main/java/com/google/devtools/build/skyframe", "//third_party:aether", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 0befcba0e8..dab80039c6 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -30,6 +30,8 @@ import com.google.devtools.build.lib.bazel.repository.MavenServerFunction; import com.google.devtools.build.lib.bazel.repository.MavenServerRepositoryFunction; import com.google.devtools.build.lib.bazel.repository.NewGitRepositoryFunction; import com.google.devtools.build.lib.bazel.repository.NewHttpArchiveFunction; +import com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryFunction; +import com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryModule; import com.google.devtools.build.lib.bazel.rules.android.AndroidNdkRepositoryFunction; import com.google.devtools.build.lib.bazel.rules.android.AndroidNdkRepositoryRule; import com.google.devtools.build.lib.bazel.rules.android.AndroidSdkRepositoryFunction; @@ -151,6 +153,7 @@ public class BazelRepositoryModule extends BlazeModule { } builder.addRuleDefinition(ruleDefinition); } + builder.addSkylarkModule(SkylarkRepositoryModule.class); } @Override @@ -171,9 +174,10 @@ public class BazelRepositoryModule extends BlazeModule { // Create the repository function everything flows through. builder.put(SkyFunctions.REPOSITORY, new RepositoryLoaderFunction()); - // Helper SkyFunctions. - builder.put(SkyFunctions.REPOSITORY_DIRECTORY, - new RepositoryDelegatorFunction(directories, repositoryHandlers, isFetch)); + builder.put( + SkyFunctions.REPOSITORY_DIRECTORY, + new RepositoryDelegatorFunction( + directories, repositoryHandlers, new SkylarkRepositoryFunction(), isFetch)); builder.put(MavenServerFunction.NAME, new MavenServerFunction(directories)); return builder.build(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/GitRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/GitRepositoryFunction.java index 7ba85c6a40..4bb282eb97 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/GitRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/GitRepositoryFunction.java @@ -33,7 +33,7 @@ import java.io.IOException; */ public class GitRepositoryFunction extends RepositoryFunction { @Override - public boolean isLocal() { + public boolean isLocal(Rule rule) { return false; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java index 5158bc6212..3952360e1e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java @@ -34,7 +34,7 @@ import java.io.IOException; */ public class HttpArchiveFunction extends RepositoryFunction { @Override - public boolean isLocal() { + public boolean isLocal(Rule rule) { return false; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java index be4e9f7cb0..44c6ca33a9 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java @@ -61,7 +61,7 @@ public class MavenJarFunction extends HttpArchiveFunction { private static final String DEFAULT_SERVER = "default"; @Override - public boolean isLocal() { + public boolean isLocal(Rule rule) { return false; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerRepositoryFunction.java index 084e073daf..4ddf40e4f5 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerRepositoryFunction.java @@ -33,7 +33,7 @@ import javax.annotation.Nullable; public class MavenServerRepositoryFunction extends RepositoryFunction { @Override - public boolean isLocal() { + public boolean isLocal(Rule rule) { return true; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java new file mode 100644 index 0000000000..5b7217fb74 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java @@ -0,0 +1,75 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.skylark; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.skylarkinterface.SkylarkCallable; +import com.google.devtools.build.lib.skylarkinterface.SkylarkModule; +import com.google.devtools.build.lib.vfs.Path; + +/** + * A Path object to be used into Skylark remote repository. + */ +@Immutable +@SkylarkModule(name = "path", doc = "A structure representing a file to be used inside a repository" +) +final class SkylarkPath { + final Path path; + + public SkylarkPath(Path path) { + this.path = path; + } + + @SkylarkCallable( + name = "basename", + structField = true, + doc = "A string giving the basename of the file." + ) + public String getBasename() { + return path.getBaseName(); + } + + @SkylarkCallable( + name = "dirname", + structField = true, + doc = "The parent directory of this file, or None if this file does not have a parent." + ) + public SkylarkPath getDirname() { + Path parentPath = path.getParentDirectory(); + return parentPath == null ? null : new SkylarkPath(parentPath); + } + + @SkylarkCallable( + name = "get_child", + doc = "Append the given path to this path and return the resulted path." + ) + public SkylarkPath getChild(String childPath) { + return new SkylarkPath(path.getChild(childPath)); + } + + @SkylarkCallable( + name = "exists", + structField = true, + doc = "Returns true if the file denoted by this path exists." + ) + public boolean exists() { + return path.exists(); + } + + @Override + public String toString() { + return path.toString(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java new file mode 100644 index 0000000000..cdd211812b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java @@ -0,0 +1,132 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.skylark; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.packages.AggregatingAttributeMapper; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException; +import com.google.devtools.build.lib.skylarkinterface.SkylarkCallable; +import com.google.devtools.build.lib.skylarkinterface.SkylarkModule; +import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject; +import com.google.devtools.build.lib.syntax.Runtime; +import com.google.devtools.build.lib.syntax.SkylarkType; +import com.google.devtools.build.lib.syntax.Type; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; + +import java.io.IOException; + +/** + * Skylark API for the repository_rule's context. + */ +@SkylarkModule( + name = "repository_ctx", + doc = + "The context of the repository rule containing" + + " helper functions and information about attributes. You get a repository_ctx object" + + " as an argument to the <code>implementation</code> function when you create a" + + " repository rule." +) +public class SkylarkRepositoryContext { + + private final Rule rule; + private final Path outputDirectory; + private final SkylarkClassObject attrObject; + + /** + * In native code, private values start with $. In Skylark, private values start with _, because + * of the grammar. + */ + private String attributeToSkylark(String oldName) { + if (!oldName.isEmpty() && (oldName.charAt(0) == '$' || oldName.charAt(0) == ':')) { + return "_" + oldName.substring(1); + } + return oldName; + } + + /** + * Create a new context (ctx) object for a skylark repository rule ({@code rule} argument). The + * environment + */ + SkylarkRepositoryContext(Rule rule, Path outputDirectory) { + this.rule = rule; + this.outputDirectory = outputDirectory; + AggregatingAttributeMapper attrs = AggregatingAttributeMapper.of(rule); + ImmutableMap.Builder<String, Object> attrBuilder = new ImmutableMap.Builder<>(); + for (String name : attrs.getAttributeNames()) { + if (!name.equals("$local")) { + Type<?> type = attrs.getAttributeType(name); + Object val = attrs.get(name, type); + attrBuilder.put( + attributeToSkylark(name), + val == null + ? Runtime.NONE + // Attribute values should be type safe + : SkylarkType.convertToSkylark(val, null)); + } + } + attrObject = new SkylarkClassObject(attrBuilder.build(), "No such attribute '%s'"); + } + + @SkylarkCallable( + name = "attr", + structField = true, + doc = + "A struct to access the values of the attributes. The values are provided by " + + "the user (if not, a default value is used)." + ) + public SkylarkClassObject getAttr() { + return attrObject; + } + + @SkylarkCallable( + name = "path", + doc = + "Returns a path from a string. If the path is relative, it will resolved relative " + + "to the output directory." + ) + public SkylarkPath path(String path) { + PathFragment pathFragment = new PathFragment(path); + if (pathFragment.isAbsolute()) { + return new SkylarkPath(outputDirectory.getFileSystem().getPath(path)); + } else { + return new SkylarkPath(outputDirectory.getRelative(pathFragment)); + } + } + + @SkylarkCallable( + name = "symlink", + doc = + "Create a symlink on the filesystem, the destination of the symlink should be in the " + + "output directory." + ) + public void symlink(SkylarkPath from, SkylarkPath to) throws RepositoryFunctionException { + try { + to.path.createSymbolicLink(from.path); + } catch (IOException e) { + throw new RepositoryFunctionException( + new IOException( + "Could not create symlink from " + from + " to " + to + ": " + e.getMessage(), e), + Transience.TRANSIENT); + } + } + + @Override + public String toString() { + return "repository_ctx[" + rule.getLabel() + "]"; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java new file mode 100644 index 0000000000..51bbe832f3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java @@ -0,0 +1,103 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.skylark; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue; +import com.google.devtools.build.lib.rules.repository.RepositoryFunction; +import com.google.devtools.build.lib.skyframe.FileValue; +import com.google.devtools.build.lib.syntax.BaseFunction; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Mutability; +import com.google.devtools.build.lib.syntax.Runtime; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.skyframe.SkyFunction.Environment; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; + +import javax.annotation.Nullable; + +/** + * A repository function to delegate work done by skylark remote repositories. + */ +public class SkylarkRepositoryFunction extends RepositoryFunction { + @Nullable + @Override + public SkyValue fetch(Rule rule, Path outputDirectory, Environment env) + throws SkyFunctionException, InterruptedException { + BaseFunction function = rule.getRuleClassObject().getConfiguredTargetFunction(); + try (Mutability mutability = Mutability.create("skylark repository")) { + com.google.devtools.build.lib.syntax.Environment buildEnv = + com.google.devtools.build.lib.syntax.Environment.builder(mutability) + .setGlobals(rule.getRuleClassObject().getRuleDefinitionEnvironment().getGlobals()) + .setSkylark() + .setEventHandler(env.getListener()) + .build(); + SkylarkRepositoryContext skylarkRepositoryContext = + new SkylarkRepositoryContext(rule, outputDirectory); + // This has side-effect, we don't care about the output. + // Also we do a lot of stuff in there, maybe blocking operations and we should certainly make + // it possible to return null and not block but it doesn't seem to be easy with Skylark + // structure as it is. + Object retValue = + function.call( + ImmutableList.<Object>of(skylarkRepositoryContext), + ImmutableMap.<String, Object>of(), + null, + buildEnv); + if (retValue != Runtime.NONE) { + throw new RepositoryFunctionException( + new EvalException( + rule.getLocation(), + "Call to repository rule " + + rule.getName() + + " returned a non-None value, None expected."), + Transience.PERSISTENT); + } + } catch (EvalException e) { + throw new RepositoryFunctionException(e, Transience.TRANSIENT); + } + + FileValue repositoryValue = getRepositoryDirectory(outputDirectory, env); + if (repositoryValue == null) { + // TODO(bazel-team): If this returns null, we unnecessarily recreate the symlink above on the + // second execution. + return null; + } + + if (!repositoryValue.isDirectory()) { + throw new RepositoryFunctionException( + new IOException(rule + " must create a directory"), Transience.TRANSIENT); + } + + return RepositoryDirectoryValue.create(outputDirectory); + } + + @Override + protected boolean isLocal(Rule rule) { + return (Boolean) rule.getAttributeContainer().getAttr("$local"); + } + + @Override + public Class<? extends RuleDefinition> getRuleDefinition() { + return null; // unused so safe to return null + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryModule.java new file mode 100644 index 0000000000..252ecdaac3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryModule.java @@ -0,0 +1,166 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.skylark; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.syntax.SkylarkType.castMap; +import static com.google.devtools.build.lib.syntax.Type.BOOLEAN; + +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Package.NameConflictException; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.PackageFactory.PackageContext; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; +import com.google.devtools.build.lib.packages.RuleFactory.InvalidRuleException; +import com.google.devtools.build.lib.rules.SkylarkAttr.Descriptor; +import com.google.devtools.build.lib.rules.SkylarkRuleClassFunctions; +import com.google.devtools.build.lib.skylarkinterface.SkylarkSignature; +import com.google.devtools.build.lib.skylarkinterface.SkylarkSignature.Param; +import com.google.devtools.build.lib.syntax.BaseFunction; +import com.google.devtools.build.lib.syntax.BuiltinFunction; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.FuncallExpression; +import com.google.devtools.build.lib.syntax.FunctionSignature; +import com.google.devtools.build.lib.syntax.Runtime; +import com.google.devtools.build.lib.syntax.SkylarkDict; +import com.google.devtools.build.lib.syntax.SkylarkSignatureProcessor; + +import java.util.Map; + +/** + * The Skylark module containing the definition of {@code repository_rule} function to define a + * skylark remote repository. + */ +public class SkylarkRepositoryModule { + + @SkylarkSignature( + name = "repository_rule", + doc = + "Creates a new repository rule. Store it in a global value, so that it can be loaded and " + + "called from the WORKSPACE file.", + returnType = BaseFunction.class, + mandatoryPositionals = { + @Param( + name = "implementation", + type = BaseFunction.class, + doc = + "the function implementing this rule, has to have exactly one parameter: " + + "<code>ctx</code>. The function is called during analysis phase for each " + + "instance of the rule." + ) + }, + optionalNamedOnly = { + @Param( + name = "attrs", + type = SkylarkDict.class, + noneable = true, + defaultValue = "None", + doc = + "dictionary to declare all the attributes of the rule. It maps from an attribute " + + "name to an attribute object (see <a href=\"#modules.attr\">attr</a> " + + "module). Attributes starting with <code>_</code> are private, and can be " + + "used to add an implicit dependency on a label to a file (a repository " + + "rule cannot depend on a generated artifact). The attribute " + + "<code>name</code> is implicitly added and must not be specified." + ), + @Param( + name = "local", + type = Boolean.class, + defaultValue = "False", + doc = + "Indicate that this rule fetches everything from the local system and should be " + + "reevaluated at every fetch." + ) + }, + useAst = true, + useEnvironment = true + ) + private static final BuiltinFunction repositoryRule = + new BuiltinFunction("repository_rule") { + @SuppressWarnings({"rawtypes", "unused"}) + // an Attribute.Builder instead of a Attribute.Builder<?> but it's OK. + public BaseFunction invoke( + BaseFunction implementation, + Object attrs, + Boolean local, + FuncallExpression ast, + com.google.devtools.build.lib.syntax.Environment funcallEnv) + throws EvalException { + funcallEnv.checkLoadingPhase("repository_rule", ast.getLocation()); + // We'll set the name later, pass the empty string for now. + Builder builder = new Builder("", RuleClassType.WORKSPACE, true); + + if (attrs != Runtime.NONE) { + for (Map.Entry<String, Descriptor> attr : + castMap(attrs, String.class, Descriptor.class, "attrs").entrySet()) { + Descriptor attrDescriptor = attr.getValue(); + String attrName = + SkylarkRuleClassFunctions.attributeToNative( + attr.getKey(), + ast.getLocation(), + attrDescriptor.getAttributeBuilder().hasLateBoundValue()); + Attribute.Builder<?> attrBuilder = attrDescriptor.getAttributeBuilder(); + builder.addOrOverrideAttribute(attrBuilder.build(attrName)); + } + } + builder.addOrOverrideAttribute(attr("$local", BOOLEAN).defaultValue(local).build()); + builder.setConfiguredTargetFunction(implementation); + builder.setRuleDefinitionEnvironment(funcallEnv); + builder.setWorkspaceOnly(); + return new RepositoryRuleFunction(builder); + } + }; + + private static final class RepositoryRuleFunction extends BaseFunction { + private final Builder builder; + + public RepositoryRuleFunction(Builder builder) { + super("repository_rule", FunctionSignature.KWARGS); + this.builder = builder; + } + + @Override + public Object call( + Object[] args, FuncallExpression ast, com.google.devtools.build.lib.syntax.Environment env) + throws EvalException, InterruptedException { + String ruleClassName = ast.getFunction().getName(); + try { + if (ruleClassName.startsWith("_")) { + throw new EvalException( + ast.getLocation(), + "Invalid rule class name '" + ruleClassName + "', cannot be private"); + } + RuleClass ruleClass = builder.build(ruleClassName); + PackageContext context = PackageFactory.getContext(env, ast); + @SuppressWarnings("unchecked") + Map<String, Object> attributeValues = (Map<String, Object>) args[0]; + return context + .getBuilder() + .externalPackageData() + .createAndAddRepositoryRule( + context.getBuilder(), ruleClass, null, attributeValues, ast); + } catch (InvalidRuleException | NameConflictException | LabelSyntaxException e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + } + } + + static { + SkylarkSignatureProcessor.configureSkylarkFunctions(SkylarkRepositoryModule.class); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryFunction.java index 04d9c5b8b8..7b3a7686ab 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryFunction.java @@ -69,7 +69,7 @@ public class AndroidNdkRepositoryFunction extends RepositoryFunction { } @Override - public boolean isLocal() { + public boolean isLocal(Rule rule) { return true; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidSdkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidSdkRepositoryFunction.java index f3f4dc9299..f9b1c65935 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidSdkRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidSdkRepositoryFunction.java @@ -34,7 +34,7 @@ import java.io.IOException; */ public class AndroidSdkRepositoryFunction extends RepositoryFunction { @Override - public boolean isLocal() { + public boolean isLocal(Rule rule) { return true; } diff --git a/src/main/java/com/google/devtools/build/lib/packages/Package.java b/src/main/java/com/google/devtools/build/lib/packages/Package.java index e01787adbb..1776ed6626 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/Package.java +++ b/src/main/java/com/google/devtools/build/lib/packages/Package.java @@ -1294,7 +1294,7 @@ public class Package { return pkg; } - protected ExternalPackageBuilder externalPackageData() { + public ExternalPackageBuilder externalPackageData() { return externalPackageData; } diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java index 055ef73bf9..36213d53fe 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java +++ b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java @@ -1398,6 +1398,13 @@ public final class PackageFactory { public MakeEnvironment.Builder getMakeEnvironment() { return pkgBuilder.getMakeEnvironment(); } + + /** + * Returns the builder of this Package. + */ + public Package.Builder getBuilder() { + return pkgBuilder; + } } private final ClassObject nativeModule; diff --git a/src/main/java/com/google/devtools/build/lib/packages/RuleClass.java b/src/main/java/com/google/devtools/build/lib/packages/RuleClass.java index 137aeb8775..8064936556 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/RuleClass.java +++ b/src/main/java/com/google/devtools/build/lib/packages/RuleClass.java @@ -559,10 +559,11 @@ public final class RuleClass { Preconditions.checkState( (type == RuleClassType.ABSTRACT) == (configuredTargetFactory == null && configuredTargetFunction == null)); - Preconditions.checkState(skylarkExecutable == (configuredTargetFunction != null)); - Preconditions.checkState(skylarkExecutable == (ruleDefinitionEnvironment != null)); - Preconditions.checkState(workspaceOnly || externalBindingsFunction == NO_EXTERNAL_BINDINGS); - + if (!workspaceOnly) { + Preconditions.checkState(skylarkExecutable == (configuredTargetFunction != null)); + Preconditions.checkState(skylarkExecutable == (ruleDefinitionEnvironment != null)); + Preconditions.checkState(externalBindingsFunction == NO_EXTERNAL_BINDINGS); + } return new RuleClass(name, skylark, skylarkExecutable, documented, publicByDefault, binaryOutput, workspaceOnly, outputsDefaultExecutable, implicitOutputsFunction, configurator, configuredTargetFactory, validityPredicate, preferredDependencyPredicate, diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleClassFunctions.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleClassFunctions.java index d9ce457863..3558cf24bf 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleClassFunctions.java +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleClassFunctions.java @@ -236,7 +236,7 @@ public class SkylarkRuleClassFunctions { * In native code, private values start with $. * In Skylark, private values start with _, because of the grammar. */ - private static String attributeToNative(String oldName, Location loc, boolean isLateBound) + public static String attributeToNative(String oldName, Location loc, boolean isLateBound) throws EvalException { if (oldName.isEmpty()) { throw new EvalException(loc, "Attribute name cannot be empty"); diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java index eb527befc9..f706daa559 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java @@ -33,7 +33,7 @@ import java.io.IOException; */ public class LocalRepositoryFunction extends RepositoryFunction { @Override - public boolean isLocal() { + public boolean isLocal(Rule rule) { return true; } diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java index 4fec7696ff..e5537c4a65 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java @@ -28,7 +28,7 @@ import com.google.devtools.build.skyframe.SkyValue; public class NewLocalRepositoryFunction extends RepositoryFunction { @Override - public boolean isLocal() { + public boolean isLocal(Rule rule) { return true; } diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java index e83433b082..c5cd01ef05 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java @@ -34,6 +34,8 @@ import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nullable; + /** * A {@link SkyFunction} that implements delegation to the correct repository fetcher. * @@ -42,19 +44,28 @@ import java.util.concurrent.atomic.AtomicBoolean; */ public class RepositoryDelegatorFunction implements SkyFunction { + // A special repository delegate used to handle Skylark remote repositories if present. + public static final String SKYLARK_DELEGATE_NAME = "$skylark"; + // Mapping of rule class name to RepositoryFunction. private final ImmutableMap<String, RepositoryFunction> handlers; + // Delegate function to handle skylark remote repositories + private final RepositoryFunction skylarkHandler; + // This is a reference to isFetch in BazelRepositoryModule, which tracks whether the current // command is a fetch. Remote repository lookups are only allowed during fetches. private final AtomicBoolean isFetch; private final BlazeDirectories directories; public RepositoryDelegatorFunction( - BlazeDirectories directories, ImmutableMap<String, RepositoryFunction> handlers, + BlazeDirectories directories, + ImmutableMap<String, RepositoryFunction> handlers, + @Nullable RepositoryFunction skylarkHandler, AtomicBoolean isFetch) { this.directories = directories; this.handlers = handlers; + this.skylarkHandler = skylarkHandler; this.isFetch = isFetch; } @@ -77,7 +88,12 @@ public class RepositoryDelegatorFunction implements SkyFunction { return null; } - RepositoryFunction handler = handlers.get(rule.getRuleClass()); + RepositoryFunction handler; + if (rule.getRuleClassObject().isSkylark()) { + handler = skylarkHandler; + } else { + handler = handlers.get(rule.getRuleClass()); + } if (handler == null) { throw new RepositoryFunctionException(new EvalException( Location.fromFile(directories.getWorkspace().getRelative("WORKSPACE")), @@ -87,7 +103,7 @@ public class RepositoryDelegatorFunction implements SkyFunction { Path repoRoot = RepositoryFunction.getExternalRepositoryDirectory(directories).getRelative(rule.getName()); - if (handler.isLocal()) { + if (handler.isLocal(rule)) { // Local repositories are always fetched because the operation is generally fast and they do // not depend on non-local data, so it does not make much sense to try to catch from across // server instances. diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java index c2f327d749..2e0843ca33 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java @@ -159,7 +159,7 @@ public abstract class RepositoryFunction { * <p>If this is false, Bazel may decide not to re-fetch the repository, for example when the * {@code --nofetch} command line option is used. */ - protected abstract boolean isLocal(); + protected abstract boolean isLocal(Rule rule); /** * Returns a block of data that must be equal for two Rules for them to be considered the same. diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD index 366fd6bd40..9a9f8caac7 100644 --- a/src/test/java/com/google/devtools/build/lib/BUILD +++ b/src/test/java/com/google/devtools/build/lib/BUILD @@ -1028,12 +1028,14 @@ java_test( java_test( name = "repository_test", srcs = glob([ - "bazel/repository/*.java", + "bazel/repository/**/*.java", ]), tags = ["rules"], test_class = "com.google.devtools.build.lib.AllTests", deps = [ + ":analysis_testutil", ":foundations_testutil", + ":packages_testutil", ":test_runner", ":testutil", "//src/main/java/com/google/devtools/build/lib:bazel-main", @@ -1042,6 +1044,7 @@ java_test( "//src/main/java/com/google/devtools/build/lib:collect", "//src/main/java/com/google/devtools/build/lib:inmemoryfs", "//src/main/java/com/google/devtools/build/lib:vfs", + "//src/main/java/com/google/devtools/build/skyframe", "//third_party:guava", "//third_party:guava-testlib", "//third_party:jsr305", diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java index f75272e4d6..cfa5572506 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java @@ -88,7 +88,8 @@ public abstract class AnalysisMock { return ImmutableMap.of( SkyFunctions.REPOSITORY_DIRECTORY, - new RepositoryDelegatorFunction(directories, repositoryHandlers, new AtomicBoolean(true)), + new RepositoryDelegatorFunction( + directories, repositoryHandlers, null, new AtomicBoolean(true)), SkyFunctions.REPOSITORY, new RepositoryLoaderFunction()); } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryIntegrationTest.java new file mode 100644 index 0000000000..36746adf28 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryIntegrationTest.java @@ -0,0 +1,172 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.skylark; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.ConfigurationCollectionFactory; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.config.ConfigurationFactory; +import com.google.devtools.build.lib.analysis.util.AnalysisMock; +import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; +import com.google.devtools.build.lib.packages.util.MockCcSupport; +import com.google.devtools.build.lib.packages.util.MockToolsConfig; +import com.google.devtools.build.lib.rules.repository.LocalRepositoryFunction; +import com.google.devtools.build.lib.rules.repository.LocalRepositoryRule; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; +import com.google.devtools.build.lib.rules.repository.RepositoryFunction; +import com.google.devtools.build.lib.rules.repository.RepositoryLoaderFunction; +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.testutil.TestRuleClassProvider; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionName; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.util.Collection; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Integration test for skylark repository not as heavyweight than shell integration tests. + */ +@RunWith(JUnit4.class) +public class SkylarkRepositoryIntegrationTest extends BuildViewTestCase { + + // The RuleClassProvider loaded with the SkylarkRepositoryModule + private ConfiguredRuleClassProvider ruleProvider = null; + // The Analysis mock injected with the SkylarkRepositoryFunction + private AnalysisMock analysisMock = null; + + /** + * Proxy to the real analysis mock to overwrite {@code #getSkyFunctions(BlazeDirectories)} to + * inject the SkylarkRepositoryFunction in the list of SkyFunctions. In Bazel, this function is + * injected by the corresponding @{code BlazeModule}. + */ + private static class CustomAnalysisMock extends AnalysisMock { + + private final AnalysisMock proxied; + + CustomAnalysisMock(AnalysisMock proxied) { + this.proxied = proxied; + } + + @Override + public ImmutableMap<SkyFunctionName, SkyFunction> getSkyFunctions( + BlazeDirectories directories) { + // Add both the local repository and the skylark repository functions + RepositoryFunction localRepositoryFunction = new LocalRepositoryFunction(); + localRepositoryFunction.setDirectories(directories); + RepositoryFunction skylarkRepositoryFunction = new SkylarkRepositoryFunction(); + skylarkRepositoryFunction.setDirectories(directories); + ImmutableMap<String, RepositoryFunction> repositoryHandlers = + ImmutableMap.of(LocalRepositoryRule.NAME, localRepositoryFunction); + + return ImmutableMap.of( + SkyFunctions.REPOSITORY_DIRECTORY, + new RepositoryDelegatorFunction( + directories, repositoryHandlers, skylarkRepositoryFunction, new AtomicBoolean(true)), + SkyFunctions.REPOSITORY, + new RepositoryLoaderFunction()); + } + + @Override + public void setupMockClient(MockToolsConfig mockToolsConfig) throws IOException { + proxied.setupMockClient(mockToolsConfig); + } + + @Override + public void setupMockWorkspaceFiles(Path embeddedBinariesRoot) throws IOException { + proxied.setupMockWorkspaceFiles(embeddedBinariesRoot); + } + + @Override + public ConfigurationFactory createConfigurationFactory() { + return proxied.createConfigurationFactory(); + } + + @Override + public ConfigurationFactory createFullConfigurationFactory() { + return proxied.createFullConfigurationFactory(); + } + + @Override + public ConfigurationCollectionFactory createConfigurationCollectionFactory() { + return proxied.createConfigurationCollectionFactory(); + } + + @Override + public Collection<String> getOptionOverrides() { + return proxied.getOptionOverrides(); + } + + @Override + public MockCcSupport ccSupport() { + return proxied.ccSupport(); + } + } + + @Override + protected AnalysisMock getAnalysisMock() { + if (analysisMock == null) { + analysisMock = new CustomAnalysisMock(super.getAnalysisMock()); + } + return analysisMock; + } + + @Override + protected ConfiguredRuleClassProvider getRuleClassProvider() { + // We inject the repository module in our test rule class provider. + if (ruleProvider == null) { + ConfiguredRuleClassProvider.Builder builder = new ConfiguredRuleClassProvider.Builder(); + TestRuleClassProvider.addStandardRules(builder); + builder.addSkylarkModule(SkylarkRepositoryModule.class); + ruleProvider = builder.build(); + } + return ruleProvider; + } + + @Test + public void testSkylarkLocalRepository() throws Exception { + // A simple test that recreates local_repository with Skylark. + scratch.file("/repo2/WORKSPACE"); + scratch.file("/repo2/bar.txt"); + scratch.file("/repo2/BUILD", "filegroup(name='bar', srcs=['bar.txt'], path='foo')"); + scratch.file( + "def.bzl", + "def _impl(ctx):", + " ctx.symlink(ctx.path(ctx.attr.path), ctx.path(''))", + "", + "repo = repository_rule(", + " implementation=_impl,", + " local=True,", + " attrs={'path': attr.string(mandatory=True)})"); + scratch.file(rootDirectory.getRelative("BUILD").getPathString()); + scratch.overwriteFile( + rootDirectory.getRelative("WORKSPACE").getPathString(), + "load('//:def.bzl', 'repo')", + "repo(name='foo', path='/repo2')"); + invalidatePackages(); + ConfiguredTarget target = getConfiguredTarget("@foo//:bar"); + Object path = target.getTarget().getAssociatedRule().getAttributeContainer().getAttr("path"); + assertThat(path).isEqualTo("foo"); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryFunctionTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryFunctionTest.java index 84a17b2794..1b7cb5f68e 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryFunctionTest.java @@ -53,7 +53,7 @@ public class RepositoryFunctionTest extends BuildViewTestCase { } @Override - protected boolean isLocal() { + protected boolean isLocal(Rule rule) { return false; } diff --git a/src/test/shell/bazel/skylark_repository_test.sh b/src/test/shell/bazel/skylark_repository_test.sh index 7b8d6338ea..c84baf42a6 100755 --- a/src/test/shell/bazel/skylark_repository_test.sh +++ b/src/test/shell/bazel/skylark_repository_test.sh @@ -216,6 +216,39 @@ EOF expect_log "Tra-la!" } +function test_skylark_local_repository() { + create_new_workspace + repo2=$new_workspace_dir + + cat > BUILD <<'EOF' +genrule(name='bar', cmd='echo foo | tee $@', outs=['bar.txt']) +EOF + + cd ${WORKSPACE_DIR} + cat > WORKSPACE <<EOF +load('/test', 'repo') +repo(name='foo', path='$repo2') +EOF + + # Our custom repository rule + cat >test.bzl <<EOF +def _impl(ctx): + ctx.symlink(ctx.path(ctx.attr.path), ctx.path("")) + +repo = repository_rule( + implementation=_impl, + local=True, + attrs={"path": attr.string(mandatory=True)}) +EOF + # Need to be in a package + cat > BUILD + + bazel build @foo//:bar >& $TEST_log || fail "Failed to build" + expect_log "foo" + cat bazel-genfiles/external/foo/bar.txt >$TEST_log + expect_log "foo" +} + function tear_down() { true } |