aboutsummaryrefslogtreecommitdiffhomepage
path: root/src
diff options
context:
space:
mode:
authorGravatar Lukacs Berki <lberki@google.com>2015-12-09 08:57:08 +0000
committerGravatar Lukacs Berki <lberki@google.com>2015-12-09 14:47:15 +0000
commit6aff7f4f2bf0c6418fb88e967653f2feeb83819e (patch)
tree2e7bf4d0e8acd2ee1359cd314a432e00319c88b3 /src
parentcdf0c157750fb663814754f77a6d41b3d36c690d (diff)
Refactor external repository support significantly to solve a number of issues.
In particular: - Separate the implementation of maven_server into a RepositoryFunction and one that creates the MavenServerValue (ideally, maven_server wouldn't exist but we'll have to make to for the time being) - Refactor the logic of determining whether an external repository needs to be re-fetched to RepositoryDelegatorFunction - Make RepositoryFunctions not be SkyFunctions anymore (they are called from RepositoryDelegatorFunction, though, who *is* a SkyFunction) - Add a Skyframe dirtiness checker that makes --nofetch RepositoryValues not be cached - Add a bunch of test cases and javadoc There is only one wart that I know of that remains: changes to BUILD files of new_* repository rules that weren't refetched when their RepositoryValue was initiall created on server restart won't take effect. This is because we don't add those BUILD files to the created RepositoryValue. This will fix itself once the ExternalFilesHelper refactoring is submitted. -- MOS_MIGRATED_REVID=109768345
Diffstat (limited to 'src')
-rw-r--r--src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java48
-rw-r--r--src/main/java/com/google/devtools/build/lib/bazel/repository/GitRepositoryFunction.java26
-rw-r--r--src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java32
-rw-r--r--src/main/java/com/google/devtools/build/lib/bazel/repository/HttpFileFunction.java32
-rw-r--r--src/main/java/com/google/devtools/build/lib/bazel/repository/HttpJarFunction.java32
-rw-r--r--src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java66
-rw-r--r--src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerFunction.java33
-rw-r--r--src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerRepositoryFunction.java53
-rw-r--r--src/main/java/com/google/devtools/build/lib/bazel/repository/NewGitRepositoryFunction.java31
-rw-r--r--src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java35
-rw-r--r--src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryFunction.java21
-rw-r--r--src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidSdkRepositoryFunction.java24
-rw-r--r--src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java22
-rw-r--r--src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java25
-rw-r--r--src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java110
-rw-r--r--src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java81
-rw-r--r--src/main/java/com/google/devtools/build/lib/skyframe/RepositoryValue.java21
-rw-r--r--src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java6
-rwxr-xr-xsrc/test/shell/bazel/git_repository_test.sh47
-rwxr-xr-xsrc/test/shell/bazel/maven_test.sh2
20 files changed, 395 insertions, 352 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
index d9e324b357..d5e7037bb9 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -31,6 +31,7 @@ import com.google.devtools.build.lib.bazel.repository.HttpJarFunction;
import com.google.devtools.build.lib.bazel.repository.JarFunction;
import com.google.devtools.build.lib.bazel.repository.MavenJarFunction;
import com.google.devtools.build.lib.bazel.repository.MavenServerFunction;
+import com.google.devtools.build.lib.bazel.repository.MavenServerRepositoryFunction;
import com.google.devtools.build.lib.bazel.repository.NewGitRepositoryFunction;
import com.google.devtools.build.lib.bazel.repository.NewHttpArchiveFunction;
import com.google.devtools.build.lib.bazel.repository.TarGzFunction;
@@ -44,6 +45,7 @@ import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule;
import com.google.devtools.build.lib.bazel.rules.workspace.HttpFileRule;
import com.google.devtools.build.lib.bazel.rules.workspace.HttpJarRule;
import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule;
+import com.google.devtools.build.lib.bazel.rules.workspace.MavenServerRule;
import com.google.devtools.build.lib.bazel.rules.workspace.NewGitRepositoryRule;
import com.google.devtools.build.lib.bazel.rules.workspace.NewHttpArchiveRule;
import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
@@ -57,16 +59,23 @@ import com.google.devtools.build.lib.runtime.BlazeCommand;
import com.google.devtools.build.lib.runtime.BlazeModule;
import com.google.devtools.build.lib.runtime.Command;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.build.lib.skyframe.RepositoryValue;
import com.google.devtools.build.lib.skyframe.SkyFunctions;
+import com.google.devtools.build.lib.skyframe.SkyValueDirtinessChecker;
import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.common.options.OptionsProvider;
import java.util.Map.Entry;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
+import javax.annotation.Nullable;
+
/**
* Adds support for fetching external code.
*/
@@ -92,6 +101,7 @@ public class BazelRepositoryModule extends BlazeModule {
.put(NewLocalRepositoryRule.NAME, new NewLocalRepositoryFunction())
.put(AndroidSdkRepositoryRule.NAME, new AndroidSdkRepositoryFunction())
.put(AndroidNdkRepositoryRule.NAME, new AndroidNdkRepositoryFunction())
+ .put(MavenServerRule.NAME, new MavenServerRepositoryFunction())
.build();
}
@@ -110,6 +120,39 @@ public class BazelRepositoryModule extends BlazeModule {
}
}
+ /**
+ * A dirtiness checker that always dirties {@link RepositoryValue}s so that if they were produced
+ * in a {@code --nofetch} build, they are re-created no subsequent {@code --fetch} builds.
+ *
+ * <p>The alternative solution would be to reify the value of the flag as a Skyframe value.
+ */
+ private static final SkyValueDirtinessChecker REPOSITORY_VALUE_CHECKER =
+ new SkyValueDirtinessChecker() {
+ @Override
+ public boolean applies(SkyKey skyKey) {
+ return skyKey.functionName().equals(SkyFunctions.REPOSITORY);
+ }
+
+ @Override
+ public SkyValue createNewValue(SkyKey key, @Nullable TimestampGranularityMonitor tsgm) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DirtyResult check(
+ SkyKey skyKey, SkyValue skyValue, @Nullable TimestampGranularityMonitor tsgm) {
+ RepositoryValue repositoryValue = (RepositoryValue) skyValue;
+ return repositoryValue.isFetchingDelayed()
+ ? DirtyResult.dirty(skyValue)
+ : DirtyResult.notDirty(skyValue);
+ }
+ };
+
+ @Override
+ public Iterable<SkyValueDirtinessChecker> getCustomDirtinessCheckers() {
+ return ImmutableList.of(REPOSITORY_VALUE_CHECKER);
+ }
+
@Override
public void initializeRuleClasses(ConfiguredRuleClassProvider.Builder builder) {
for (Entry<String, RepositoryFunction> handler : repositoryHandlers.entrySet()) {
@@ -139,11 +182,6 @@ public class BazelRepositoryModule extends BlazeModule {
public ImmutableMap<SkyFunctionName, SkyFunction> getSkyFunctions(BlazeDirectories directories) {
ImmutableMap.Builder<SkyFunctionName, SkyFunction> builder = ImmutableMap.builder();
- // Bazel-specific repository downloaders.
- for (RepositoryFunction handler : repositoryHandlers.values()) {
- builder.put(handler.getSkyFunctionName(), handler);
- }
-
// Create the delegator everything flows through.
builder.put(SkyFunctions.REPOSITORY,
new RepositoryDelegatorFunction(directories, repositoryHandlers, isFetch));
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/GitRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/GitRepositoryFunction.java
index 2b8a4ebadc..683d05ebc6 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/GitRepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/GitRepositoryFunction.java
@@ -16,16 +16,14 @@ package com.google.devtools.build.lib.bazel.repository;
import com.google.devtools.build.lib.analysis.RuleDefinition;
import com.google.devtools.build.lib.bazel.rules.workspace.GitRepositoryRule;
-import com.google.devtools.build.lib.cmdline.PackageIdentifier.RepositoryName;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
import com.google.devtools.build.lib.skyframe.RepositoryValue;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
-import com.google.devtools.build.skyframe.SkyFunctionName;
-import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import java.io.IOException;
@@ -35,18 +33,14 @@ import java.io.IOException;
*/
public class GitRepositoryFunction extends RepositoryFunction {
@Override
- public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
- RepositoryName repositoryName = (RepositoryName) skyKey.argument();
- Rule rule = RepositoryFunction.getRule(repositoryName, GitRepositoryRule.NAME, env);
- if (rule == null) {
- return null;
- }
+ public boolean isLocal() {
+ return false;
+ }
+ @Override
+ public SkyValue fetch(Rule rule, Environment env)
+ throws SkyFunctionException {
Path outputDirectory = getExternalRepositoryDirectory().getRelative(rule.getName());
- if (isFilesystemUpToDate(rule, NO_RULE_SPECIFIC_DATA)) {
- return RepositoryValue.create(outputDirectory);
- }
-
createDirectory(outputDirectory, rule);
try {
@@ -59,7 +53,6 @@ public class GitRepositoryFunction extends RepositoryFunction {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
- writeMarkerFile(rule, NO_RULE_SPECIFIC_DATA);
return RepositoryValue.create(outputDirectory);
}
@@ -74,11 +67,6 @@ public class GitRepositoryFunction extends RepositoryFunction {
}
@Override
- public SkyFunctionName getSkyFunctionName() {
- return SkyFunctionName.create(GitRepositoryRule.NAME.toUpperCase());
- }
-
- @Override
public Class<? extends RuleDefinition> getRuleDefinition() {
return GitRepositoryRule.class;
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java
index ca5c1b1020..ad76306fcb 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java
@@ -16,15 +16,13 @@ package com.google.devtools.build.lib.bazel.repository;
import com.google.devtools.build.lib.analysis.RuleDefinition;
import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule;
-import com.google.devtools.build.lib.cmdline.PackageIdentifier.RepositoryName;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
import com.google.devtools.build.lib.skyframe.RepositoryValue;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
-import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
@@ -34,25 +32,9 @@ import java.io.IOException;
* Downloads a file over HTTP.
*/
public class HttpArchiveFunction extends RepositoryFunction {
-
@Override
- public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
- RepositoryName repositoryName = (RepositoryName) skyKey.argument();
- Rule rule = RepositoryFunction.getRule(repositoryName, HttpArchiveRule.NAME, env);
- if (rule == null) {
- return null;
- }
-
- if (isFilesystemUpToDate(rule, NO_RULE_SPECIFIC_DATA)) {
- return RepositoryValue.create(getExternalRepositoryDirectory().getRelative(rule.getName()));
- }
-
- SkyValue result = compute(env, rule);
- if (result != null) {
- writeMarkerFile(rule, NO_RULE_SPECIFIC_DATA);
- }
-
- return result;
+ public boolean isLocal() {
+ return false;
}
protected void createDirectory(Path path)
@@ -64,7 +46,8 @@ public class HttpArchiveFunction extends RepositoryFunction {
}
}
- protected SkyValue compute(Environment env, Rule rule)
+ @Override
+ public SkyValue fetch(Rule rule, Environment env)
throws RepositoryFunctionException {
// The output directory is always under .external-repository (to stay out of the way of
// artifacts from this repository) and uses the rule's name to avoid conflicts with other
@@ -105,11 +88,6 @@ public class HttpArchiveFunction extends RepositoryFunction {
}
@Override
- public SkyFunctionName getSkyFunctionName() {
- return SkyFunctionName.create(HttpArchiveRule.NAME.toUpperCase());
- }
-
- @Override
public Class<? extends RuleDefinition> getRuleDefinition() {
return HttpArchiveRule.class;
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpFileFunction.java
index 3931873be1..0b287bb397 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpFileFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpFileFunction.java
@@ -16,17 +16,11 @@ package com.google.devtools.build.lib.bazel.repository;
import com.google.devtools.build.lib.analysis.RuleDefinition;
import com.google.devtools.build.lib.bazel.rules.workspace.HttpFileRule;
-import com.google.devtools.build.lib.cmdline.PackageIdentifier.RepositoryName;
import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
import com.google.devtools.build.lib.packages.Rule;
-import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
-import com.google.devtools.build.lib.skyframe.RepositoryValue;
import com.google.devtools.build.lib.syntax.Type;
import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.skyframe.SkyFunctionException;
-import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
-import com.google.devtools.build.skyframe.SkyValue;
import java.io.IOException;
@@ -34,27 +28,6 @@ import java.io.IOException;
* Downloads a jar file from a URL.
*/
public class HttpFileFunction extends HttpArchiveFunction {
-
- @Override
- public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
- RepositoryName repositoryName = (RepositoryName) skyKey.argument();
- Rule rule = RepositoryFunction.getRule(repositoryName, HttpFileRule.NAME, env);
- if (rule == null) {
- return null;
- }
-
- if (isFilesystemUpToDate(rule, NO_RULE_SPECIFIC_DATA)) {
- return RepositoryValue.create(getExternalRepositoryDirectory().getRelative(rule.getName()));
- }
-
- SkyValue result = compute(env, rule);
- if (!env.valuesMissing()) {
- writeMarkerFile(rule, NO_RULE_SPECIFIC_DATA);
- }
-
- return result;
- }
-
@Override
protected SkyKey decompressorValueKey(Rule rule, Path downloadPath, Path outputDirectory)
throws IOException {
@@ -71,11 +44,6 @@ public class HttpFileFunction extends HttpArchiveFunction {
}
@Override
- public SkyFunctionName getSkyFunctionName() {
- return SkyFunctionName.create(HttpFileRule.NAME.toUpperCase());
- }
-
- @Override
public Class<? extends RuleDefinition> getRuleDefinition() {
return HttpFileRule.class;
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpJarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpJarFunction.java
index 0efca38268..a3f4d080b3 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpJarFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpJarFunction.java
@@ -16,15 +16,9 @@ package com.google.devtools.build.lib.bazel.repository;
import com.google.devtools.build.lib.analysis.RuleDefinition;
import com.google.devtools.build.lib.bazel.rules.workspace.HttpJarRule;
-import com.google.devtools.build.lib.cmdline.PackageIdentifier.RepositoryName;
import com.google.devtools.build.lib.packages.Rule;
-import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
-import com.google.devtools.build.lib.skyframe.RepositoryValue;
import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.skyframe.SkyFunctionException;
-import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
-import com.google.devtools.build.skyframe.SkyValue;
import java.io.IOException;
@@ -32,27 +26,6 @@ import java.io.IOException;
* Downloads a jar file from a URL.
*/
public class HttpJarFunction extends HttpArchiveFunction {
-
- @Override
- public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
- RepositoryName repositoryName = (RepositoryName) skyKey.argument();
- Rule rule = RepositoryFunction.getRule(repositoryName, HttpJarRule.NAME, env);
- if (rule == null) {
- return null;
- }
-
- if (isFilesystemUpToDate(rule, NO_RULE_SPECIFIC_DATA)) {
- return RepositoryValue.create(getExternalRepositoryDirectory().getRelative(rule.getName()));
- }
-
- SkyValue result = compute(env, rule);
- if (!env.valuesMissing()) {
- writeMarkerFile(rule, NO_RULE_SPECIFIC_DATA);
- }
-
- return result;
- }
-
@Override
protected SkyKey decompressorValueKey(Rule rule, Path downloadPath, Path outputDirectory)
throws IOException {
@@ -65,11 +38,6 @@ public class HttpJarFunction extends HttpArchiveFunction {
}
@Override
- public SkyFunctionName getSkyFunctionName() {
- return SkyFunctionName.create(HttpJarRule.NAME.toUpperCase());
- }
-
- @Override
public Class<? extends RuleDefinition> getRuleDefinition() {
return HttpJarRule.class;
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java
index ba929da820..3b519a4300 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java
@@ -15,7 +15,6 @@
package com.google.devtools.build.lib.bazel.repository;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Ascii;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -24,7 +23,6 @@ import com.google.common.hash.Hashing;
import com.google.devtools.build.lib.analysis.RuleDefinition;
import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule;
import com.google.devtools.build.lib.cmdline.PackageIdentifier.RepositoryName;
-import com.google.devtools.build.lib.events.Location;
import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
import com.google.devtools.build.lib.packages.AttributeMap;
import com.google.devtools.build.lib.packages.Rule;
@@ -34,9 +32,8 @@ import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.Type;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
-import com.google.devtools.build.skyframe.SkyFunctionName;
-import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import org.apache.maven.settings.Server;
@@ -61,58 +58,62 @@ import javax.annotation.Nullable;
* Implementation of maven_jar.
*/
public class MavenJarFunction extends HttpArchiveFunction {
-
private static final String DEFAULT_SERVER = "default";
@Override
- public SkyValue compute(SkyKey skyKey, Environment env) throws RepositoryFunctionException {
- RepositoryName repositoryName = (RepositoryName) skyKey.argument();
- Rule rule = RepositoryFunction.getRule(repositoryName, MavenJarRule.NAME, env);
- if (rule == null) {
+ public boolean isLocal() {
+ return false;
+ }
+
+ @Override
+ protected byte[] getRuleSpecificMarkerData(Rule rule, Environment env)
+ throws RepositoryFunctionException {
+ MavenServerValue serverValue = getServer(rule, env);
+ if (env.valuesMissing()) {
return null;
}
- MavenServerValue serverValue;
+ return new Fingerprint()
+ .addString(serverValue.getUrl())
+ .addBytes(serverValue.getSettingsFingerprint())
+ .digestAndReset();
+ }
+
+ private MavenServerValue getServer(Rule rule, Environment env)
+ throws RepositoryFunctionException {
AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule);
boolean hasRepository = mapper.has("repository", Type.STRING)
&& !mapper.get("repository", Type.STRING).isEmpty();
boolean hasServer = mapper.has("server", Type.STRING)
&& !mapper.get("server", Type.STRING).isEmpty();
+
if (hasRepository && hasServer) {
throw new RepositoryFunctionException(new EvalException(
- Location.fromFile(getWorkspace().getRelative("WORKSPACE")), rule + " specifies both "
+ rule.getLocation(), rule + " specifies both "
+ "'repository' and 'server', which are mutually exclusive options"),
Transience.PERSISTENT);
} else if (hasRepository) {
- serverValue = MavenServerValue.createFromUrl(mapper.get("repository", Type.STRING));
+ return MavenServerValue.createFromUrl(mapper.get("repository", Type.STRING));
} else {
String serverName = DEFAULT_SERVER;
if (hasServer) {
serverName = mapper.get("server", Type.STRING);
}
- serverValue = (MavenServerValue) env.getValue(MavenServerValue.key(serverName));
- if (serverValue == null) {
- return null;
- }
+ return (MavenServerValue) env.getValue(MavenServerValue.key(serverName));
}
+ }
- byte[] serverData = new Fingerprint()
- .addString(serverValue.getUrl())
- .addBytes(serverValue.getSettingsFingerprint())
- .digestAndReset();
-
- if (isFilesystemUpToDate(rule, serverData)) {
- return RepositoryValue.create(getExternalRepositoryDirectory().getRelative(rule.getName()));
+ @Override
+ public SkyValue fetch(Rule rule, Environment env)
+ throws RepositoryFunctionException {
+ AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule);
+ MavenServerValue serverValue = getServer(rule, env);
+ if (env.valuesMissing()) {
+ return null;
}
-
MavenDownloader downloader = createMavenDownloader(mapper, serverValue);
- SkyValue result = createOutputTree(downloader, env);
- if (!env.valuesMissing()) {
- writeMarkerFile(rule, serverData);
- }
-
- return result;
+ return createOutputTree(downloader, env);
}
@VisibleForTesting
@@ -153,11 +154,6 @@ public class MavenJarFunction extends HttpArchiveFunction {
return RepositoryValue.create(value.getDirectory());
}
- @Override
- public SkyFunctionName getSkyFunctionName() {
- return SkyFunctionName.create(Ascii.toUpperCase(MavenJarRule.NAME));
- }
-
/**
* @see RepositoryFunction#getRule(RepositoryName, String, Environment)
*/
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerFunction.java
index e93550905a..c0b9414c32 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerFunction.java
@@ -17,18 +17,20 @@ package com.google.devtools.build.lib.bazel.repository;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
-import com.google.devtools.build.lib.analysis.RuleDefinition;
import com.google.devtools.build.lib.bazel.rules.workspace.MavenServerRule;
import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
import com.google.devtools.build.lib.packages.Package;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
+import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException;
import com.google.devtools.build.lib.skyframe.FileValue;
import com.google.devtools.build.lib.syntax.Type;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
@@ -51,20 +53,22 @@ import javax.annotation.Nullable;
/**
* Implementation of maven_repository.
*/
-public class MavenServerFunction extends RepositoryFunction {
+public class MavenServerFunction implements SkyFunction {
public static final SkyFunctionName NAME = SkyFunctionName.create("MAVEN_SERVER_FUNCTION");
private static final String USER_KEY = "user";
private static final String SYSTEM_KEY = "system";
+ private final BlazeDirectories directories;
+
public MavenServerFunction(BlazeDirectories directories) {
- setDirectories(directories);
+ this.directories = directories;
}
@Nullable
- @Override
- public SkyValue compute(SkyKey skyKey, Environment env) throws RepositoryFunctionException {
- String repository = skyKey.argument().toString();
+ public SkyValue compute(SkyKey skyKey, Environment env)
+ throws SkyFunctionException {
+ String repository = (String) skyKey.argument();
Package externalPackage = RepositoryFunction.getExternalPackage(env);
if (externalPackage == null) {
return null;
@@ -96,7 +100,7 @@ public class MavenServerFunction extends RepositoryFunction {
} else {
PathFragment settingsFilePath = new PathFragment(mapper.get("settings_file", Type.STRING));
RootedPath settingsPath = RootedPath.toRootedPath(
- getWorkspace().getRelative(settingsFilePath), PathFragment.EMPTY_FRAGMENT);
+ directories.getWorkspace().getRelative(settingsFilePath), PathFragment.EMPTY_FRAGMENT);
FileValue fileValue = (FileValue) env.getValue(FileValue.key(settingsPath));
if (fileValue == null) {
return null;
@@ -175,7 +179,7 @@ public class MavenServerFunction extends RepositoryFunction {
if (m2Home != null) {
PathFragment mavenInstallSettings = new PathFragment(m2Home).getRelative("conf/settings.xml");
systemKey = FileValue.key(
- RootedPath.toRootedPath(getWorkspace().getRelative(mavenInstallSettings),
+ RootedPath.toRootedPath(directories.getWorkspace().getRelative(mavenInstallSettings),
PathFragment.EMPTY_FRAGMENT));
settingsFilesBuilder.add(systemKey);
}
@@ -185,7 +189,8 @@ public class MavenServerFunction extends RepositoryFunction {
SkyKey userKey = null;
if (userHome != null) {
PathFragment userSettings = new PathFragment(userHome).getRelative(".m2/settings.xml");
- userKey = FileValue.key(RootedPath.toRootedPath(getWorkspace().getRelative(userSettings),
+ userKey = FileValue.key(RootedPath.toRootedPath(
+ directories.getWorkspace().getRelative(userSettings),
PathFragment.EMPTY_FRAGMENT));
settingsFilesBuilder.add(userKey);
}
@@ -209,13 +214,9 @@ public class MavenServerFunction extends RepositoryFunction {
return settingsBuilder.build();
}
+ @Nullable
@Override
- public SkyFunctionName getSkyFunctionName() {
- return NAME;
- }
-
- @Override
- public Class<? extends RuleDefinition> getRuleDefinition() {
- return MavenServerRule.class;
+ public String extractTag(SkyKey skyKey) {
+ return null;
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerRepositoryFunction.java
new file mode 100644
index 0000000000..f5e220fae5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerRepositoryFunction.java
@@ -0,0 +1,53 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.bazel.rules.workspace.MavenServerRule;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import javax.annotation.Nullable;
+
+/**
+ * Stub {@link RepositoryFunction} for {@code maven_server} rules.
+ */
+public class MavenServerRepositoryFunction extends RepositoryFunction {
+
+ @Override
+ public boolean isLocal() {
+ return true;
+ }
+
+ @Nullable
+ @Override
+ public SkyValue fetch(Rule rule, Environment env)
+ throws SkyFunctionException, InterruptedException {
+ throw new RepositoryFunctionException(new EvalException(
+ rule.getLocation(),
+ "maven_server rule '" + rule.getName() + "' does not represent an actual repository"),
+ Transience.PERSISTENT);
+ }
+
+ @Override
+ public Class<? extends RuleDefinition> getRuleDefinition() {
+ return MavenServerRule.class;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/NewGitRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewGitRepositoryFunction.java
index 6303880f0b..2c32783b35 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/NewGitRepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewGitRepositoryFunction.java
@@ -14,17 +14,11 @@
package com.google.devtools.build.lib.bazel.repository;
-import com.google.devtools.build.lib.bazel.rules.workspace.NewGitRepositoryRule;
-import com.google.devtools.build.lib.cmdline.PackageIdentifier.RepositoryName;
import com.google.devtools.build.lib.packages.Rule;
-import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
-import com.google.devtools.build.lib.skyframe.FileValue;
-import com.google.devtools.build.lib.skyframe.RepositoryValue;
import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
-import com.google.devtools.build.skyframe.SkyFunctionName;
-import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import java.io.IOException;
@@ -34,28 +28,9 @@ import java.io.IOException;
*/
public class NewGitRepositoryFunction extends GitRepositoryFunction {
@Override
- public SkyFunctionName getSkyFunctionName() {
- return SkyFunctionName.create(NewGitRepositoryRule.NAME.toUpperCase());
- }
-
- @Override
- public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
- RepositoryName repositoryName = (RepositoryName) skyKey.argument();
- Rule rule = RepositoryFunction.getRule(repositoryName, NewGitRepositoryRule.NAME, env);
- if (rule == null) {
- return null;
- }
-
+ public SkyValue fetch(Rule rule, Environment env)
+ throws SkyFunctionException {
Path outputDirectory = getExternalRepositoryDirectory().getRelative(rule.getName());
- if (isFilesystemUpToDate(rule, NO_RULE_SPECIFIC_DATA)) {
- FileValue buildFileValue = getBuildFileValue(rule, env);
- if (env.valuesMissing()) {
- return null;
- }
-
- return RepositoryValue.createNew(outputDirectory, buildFileValue);
- }
-
createDirectory(outputDirectory, rule);
try {
HttpDownloadValue value = (HttpDownloadValue) env.getValueOrThrow(
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java
index 6c0297a5b2..0395f2b2c4 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java
@@ -14,19 +14,13 @@
package com.google.devtools.build.lib.bazel.repository;
-import com.google.devtools.build.lib.bazel.rules.workspace.NewHttpArchiveRule;
-import com.google.devtools.build.lib.cmdline.PackageIdentifier.RepositoryName;
import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
import com.google.devtools.build.lib.packages.Rule;
-import com.google.devtools.build.lib.skyframe.FileValue;
-import com.google.devtools.build.lib.skyframe.RepositoryValue;
import com.google.devtools.build.lib.syntax.Type;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
-import com.google.devtools.build.skyframe.SkyFunctionName;
-import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import java.io.IOException;
@@ -39,29 +33,11 @@ import javax.annotation.Nullable;
*/
public class NewHttpArchiveFunction extends HttpArchiveFunction {
- @Override
- public SkyFunctionName getSkyFunctionName() {
- return SkyFunctionName.create(NewHttpArchiveRule.NAME);
- }
-
@Nullable
@Override
- public SkyValue compute(SkyKey skyKey, SkyFunction.Environment env)
+ public SkyValue fetch(Rule rule, Environment env)
throws RepositoryFunctionException {
- RepositoryName repositoryName = (RepositoryName) skyKey.argument();
- Rule rule = getRule(repositoryName, NewHttpArchiveRule.NAME, env);
- if (rule == null) {
- return null;
- }
Path outputDirectory = getExternalRepositoryDirectory().getRelative(rule.getName());
- if (isFilesystemUpToDate(rule, NO_RULE_SPECIFIC_DATA)) {
- FileValue buildFileValue = getBuildFileValue(rule, env);
- if (env.valuesMissing()) {
- return null;
- }
-
- return RepositoryValue.createNew(outputDirectory, buildFileValue);
- }
try {
FileSystemUtils.createDirectoryAndParents(outputDirectory);
@@ -109,11 +85,6 @@ public class NewHttpArchiveFunction extends HttpArchiveFunction {
// Add WORKSPACE and BUILD files.
createWorkspaceFile(decompressed.getDirectory(), rule);
- SkyValue result = symlinkBuildFile(rule, outputDirectory, env);
- if (!env.valuesMissing()) {
- writeMarkerFile(rule, NO_RULE_SPECIFIC_DATA);
- }
-
- return result;
+ return symlinkBuildFile(rule, outputDirectory, env);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryFunction.java
index fe13ce8a05..6b1c9c2727 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryFunction.java
@@ -22,7 +22,6 @@ import com.google.devtools.build.lib.bazel.rules.android.ndkcrosstools.NdkPaths;
import com.google.devtools.build.lib.bazel.rules.android.ndkcrosstools.NdkRelease;
import com.google.devtools.build.lib.bazel.rules.android.ndkcrosstools.StlImpl;
import com.google.devtools.build.lib.bazel.rules.android.ndkcrosstools.StlImpls;
-import com.google.devtools.build.lib.cmdline.PackageIdentifier.RepositoryName;
import com.google.devtools.build.lib.packages.AttributeMap;
import com.google.devtools.build.lib.packages.NonconfigurableAttributeMapper;
import com.google.devtools.build.lib.packages.Rule;
@@ -39,9 +38,9 @@ import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.CToolchain;
import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.CrosstoolRelease;
import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.DefaultCpuToolchain;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
-import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
@@ -69,14 +68,13 @@ public class AndroidNdkRepositoryFunction extends RepositoryFunction {
}
@Override
- public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
-
- RepositoryName repositoryName = (RepositoryName) skyKey.argument();
- Rule rule = getRule(repositoryName, AndroidNdkRepositoryRule.NAME, env);
- if (rule == null) {
- return null;
- }
+ public boolean isLocal() {
+ return true;
+ }
+ @Override
+ public SkyValue fetch(Rule rule, Environment env)
+ throws SkyFunctionException {
Path outputDirectory = prepareLocalRepositorySymlinkTree(rule, env);
PathFragment pathFragment = getTargetPath(rule);
Path ndkSymlinkTreeDirectory = outputDirectory.getRelative("ndk");
@@ -127,11 +125,6 @@ public class AndroidNdkRepositoryFunction extends RepositoryFunction {
}
@Override
- public SkyFunctionName getSkyFunctionName() {
- return SkyFunctionName.create(AndroidNdkRepositoryRule.NAME.toUpperCase());
- }
-
- @Override
public Class<? extends RuleDefinition> getRuleDefinition() {
return AndroidNdkRepositoryRule.class;
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidSdkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidSdkRepositoryFunction.java
index 00e53cbfef..9c13cea902 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidSdkRepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidSdkRepositoryFunction.java
@@ -14,7 +14,6 @@
package com.google.devtools.build.lib.bazel.rules.android;
import com.google.devtools.build.lib.analysis.RuleDefinition;
-import com.google.devtools.build.lib.cmdline.PackageIdentifier.RepositoryName;
import com.google.devtools.build.lib.packages.AttributeMap;
import com.google.devtools.build.lib.packages.NonconfigurableAttributeMapper;
import com.google.devtools.build.lib.packages.Rule;
@@ -23,9 +22,8 @@ import com.google.devtools.build.lib.syntax.Type;
import com.google.devtools.build.lib.util.ResourceFileLoader;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyFunctionException;
-import com.google.devtools.build.skyframe.SkyFunctionName;
-import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import java.io.IOException;
@@ -35,13 +33,13 @@ import java.io.IOException;
*/
public class AndroidSdkRepositoryFunction extends RepositoryFunction {
@Override
- public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
- RepositoryName repositoryName = (RepositoryName) skyKey.argument();
- Rule rule = getRule(repositoryName, AndroidSdkRepositoryRule.NAME, env);
- if (rule == null) {
- return null;
- }
+ public boolean isLocal() {
+ return true;
+ }
+ @Override
+ public SkyValue fetch(Rule rule, Environment env)
+ throws SkyFunctionException {
Path outputDirectory = prepareLocalRepositorySymlinkTree(rule, env);
PathFragment pathFragment = getTargetPath(rule);
@@ -71,14 +69,6 @@ public class AndroidSdkRepositoryFunction extends RepositoryFunction {
return writeBuildFile(outputDirectory, buildFile);
}
- /**
- * @see RepositoryFunction#getRule(RepositoryName, String, Environment)
- */
- @Override
- public SkyFunctionName getSkyFunctionName() {
- return SkyFunctionName.create(AndroidSdkRepositoryRule.NAME.toUpperCase());
- }
-
@Override
public Class<? extends RuleDefinition> getRuleDefinition() {
return AndroidSdkRepositoryRule.class;
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java
index 40a0ee7cb7..bbf22e9b5c 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java
@@ -15,7 +15,6 @@
package com.google.devtools.build.lib.rules.repository;
import com.google.devtools.build.lib.analysis.RuleDefinition;
-import com.google.devtools.build.lib.cmdline.PackageIdentifier.RepositoryName;
import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.skyframe.FileValue;
@@ -26,10 +25,9 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
-import com.google.devtools.build.skyframe.SkyFunctionName;
-import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import java.io.IOException;
@@ -38,15 +36,14 @@ import java.io.IOException;
* Access a repository on the local filesystem.
*/
public class LocalRepositoryFunction extends RepositoryFunction {
-
@Override
- public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
- RepositoryName repositoryName = (RepositoryName) skyKey.argument();
- Rule rule = RepositoryFunction.getRule(repositoryName, LocalRepositoryRule.NAME, env);
- if (rule == null) {
- return null;
- }
+ public boolean isLocal() {
+ return true;
+ }
+ @Override
+ public SkyValue fetch(Rule rule, Environment env)
+ throws SkyFunctionException {
AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule);
String path = mapper.get("path", Type.STRING);
PathFragment pathFragment = new PathFragment(path);
@@ -85,11 +82,6 @@ public class LocalRepositoryFunction extends RepositoryFunction {
}
@Override
- public SkyFunctionName getSkyFunctionName() {
- return SkyFunctionName.create(LocalRepositoryRule.NAME.toUpperCase());
- }
-
- @Override
public Class<? extends RuleDefinition> getRuleDefinition() {
return LocalRepositoryRule.class;
}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java
index 125900b989..116c72ed7e 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java
@@ -15,28 +15,25 @@
package com.google.devtools.build.lib.rules.repository;
import com.google.devtools.build.lib.analysis.RuleDefinition;
-import com.google.devtools.build.lib.cmdline.PackageIdentifier.RepositoryName;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyFunctionException;
-import com.google.devtools.build.skyframe.SkyFunctionName;
-import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
/**
* Create a repository from a directory on the local filesystem.
*/
public class NewLocalRepositoryFunction extends RepositoryFunction {
-
@Override
- public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
- RepositoryName repositoryName = (RepositoryName) skyKey.argument();
- Rule rule = getRule(repositoryName, NewLocalRepositoryRule.NAME, env);
- if (rule == null) {
- return null;
- }
+ public boolean isLocal() {
+ return true;
+ }
+ @Override
+ public SkyValue fetch(Rule rule, Environment env)
+ throws SkyFunctionException {
Path outputDirectory = prepareLocalRepositorySymlinkTree(rule, env);
PathFragment pathFragment = getTargetPath(rule);
@@ -52,14 +49,6 @@ public class NewLocalRepositoryFunction extends RepositoryFunction {
return symlinkBuildFile(rule, outputDirectory, env);
}
- /**
- * @see RepositoryFunction#getRule(RepositoryName, String, Environment)
- */
- @Override
- public SkyFunctionName getSkyFunctionName() {
- return SkyFunctionName.create(NewLocalRepositoryRule.NAME.toUpperCase());
- }
-
@Override
public Class<? extends RuleDefinition> getRuleDefinition() {
return NewLocalRepositoryRule.class;
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java
index e29769ec24..a79b1d848b 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java
@@ -17,8 +17,8 @@ package com.google.devtools.build.lib.rules.repository;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.cmdline.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.Location;
-import com.google.devtools.build.lib.packages.NoSuchPackageException;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException;
import com.google.devtools.build.lib.skyframe.FileValue;
@@ -35,11 +35,14 @@ import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
/**
- * Implements delegation to the correct repository fetcher.
+ * A {@link SkyFunction} that implements delegation to the correct repository fetcher.
+ *
+ * <p>Each repository in the WORKSPACE file is represented by a {@link SkyValue} that is computed
+ * by this function.
*/
public class RepositoryDelegatorFunction implements SkyFunction {
- // Mapping of rule class name to SkyFunction.
+ // Mapping of rule class name to RepositoryFunction.
private final ImmutableMap<String, RepositoryFunction> handlers;
// This is a reference to isFetch in BazelRepositoryModule, which tracks whether the current
@@ -56,7 +59,8 @@ public class RepositoryDelegatorFunction implements SkyFunction {
}
@Override
- public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ public SkyValue compute(SkyKey skyKey, Environment env)
+ throws SkyFunctionException, InterruptedException {
RepositoryName repositoryName = (RepositoryName) skyKey.argument();
Rule rule = RepositoryFunction
.getRule(repositoryName, null, env);
@@ -64,45 +68,79 @@ public class RepositoryDelegatorFunction implements SkyFunction {
return null;
}
- // If Bazel isn't running a fetch command, we shouldn't be able to download anything. To
- // prevent having to rerun fetch on server restart, we check if the external repository
- // directory already exists and, if it does, just use that.
- if (!isFetch.get()) {
- FileValue repoRoot = RepositoryFunction.getRepositoryDirectory(
- RepositoryFunction
- .getExternalRepositoryDirectory(directories)
- .getRelative(rule.getName()), env);
- if (repoRoot == null) {
- return null;
- }
- Path repoPath = repoRoot.realRootedPath().asPath();
- if (!repoPath.exists()) {
- throw new RepositoryFunctionException(new IOException(
- "to fix, run\n\tbazel fetch //...\nExternal repository " + repositoryName
- + " not found"),
- Transience.TRANSIENT);
- }
- return RepositoryValue.create(repoPath);
- }
-
RepositoryFunction handler = handlers.get(rule.getRuleClass());
if (handler == null) {
throw new RepositoryFunctionException(new EvalException(
Location.fromFile(directories.getWorkspace().getRelative("WORKSPACE")),
"Could not find handler for " + rule), Transience.PERSISTENT);
}
- SkyKey key = new SkyKey(handler.getSkyFunctionName(), repositoryName);
-
- try {
- return env.getValueOrThrow(
- key, NoSuchPackageException.class, IOException.class, EvalException.class);
- } catch (NoSuchPackageException e) {
- throw new RepositoryFunctionException(e, Transience.PERSISTENT);
- } catch (IOException e) {
- throw new RepositoryFunctionException(e, Transience.PERSISTENT);
- } catch (EvalException e) {
- throw new RepositoryFunctionException(e, Transience.PERSISTENT);
+
+ if (handler.isLocal()) {
+ // Local repositories are always fetched because the operation is generally fast and they do
+ // not depend on non-local data, so it does not make much sense to try to catch from across
+ // server instances.
+ return handler.fetch(rule, env);
+ }
+
+ // We check the repository root for existence here, but we can't depend on the FileValue,
+ // because it's possible that we eventually create that directory in which case the FileValue
+ // and the state of the file system would be inconsistent.
+ Path repoRoot =
+ RepositoryFunction.getExternalRepositoryDirectory(directories).getRelative(rule.getName());
+
+ byte[] ruleSpecificData = handler.getRuleSpecificMarkerData(rule, env);
+ boolean markerUpToDate = handler.isFilesystemUpToDate(rule, ruleSpecificData);
+ if (markerUpToDate && repoRoot.exists()) {
+ // Now that we know that it exists, we can declare a Skyframe dependency on the repository
+ // root.
+ FileValue repoRootValue = RepositoryFunction.getRepositoryDirectory(repoRoot, env);
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ // NB: This returns the wrong repository value for non-local new_* repository functions.
+ // This should sort itself out automatically once the ExternalFilesHelper refactoring is
+ // finally submitted.
+ return RepositoryValue.create(repoRootValue.realRootedPath().asPath());
+ }
+
+ if (isFetch.get()) {
+ // Fetching enabled, go ahead.
+ SkyValue result = handler.fetch(rule, env);
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ handler.writeMarkerFile(rule, ruleSpecificData);
+ return result;
+ }
+
+ if (!repoRoot.exists()) {
+ // The repository isn't on the file system, there is nothing we can do.
+ throw new RepositoryFunctionException(new IOException(
+ "to fix, run\n\tbazel fetch //...\nExternal repository " + repositoryName
+ + " not found and fetching repositories is disabled."),
+ Transience.TRANSIENT);
+ }
+
+ // Declare a Skyframe dependency so that this is re-evaluated when something happens to the
+ // directory.
+ FileValue repoRootValue = RepositoryFunction.getRepositoryDirectory(repoRoot, env);
+ if (env.valuesMissing()) {
+ return null;
}
+
+ // Try to build with whatever is on the file system and emit a warning.
+ env.getListener().handle(Event.warn(rule.getLocation(), String.format(
+ "External repository '%s' is not up-to-date and fetching is disabled. To update, "
+ + "run the build without the '--nofetch' command line option.",
+ rule.getName())));
+
+ // NB: This returns the wrong repository value for non-local new_* repository functions because
+ // overlaidBuildFile won't be set.
+ // TODO(lberki): overlaidBuildFile can now probably be removed. Try to excise it write a test
+ // that makes sure this works as expected.
+ return RepositoryValue.fetchingDelayed(repoRootValue.realRootedPath().asPath());
}
@Override
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
index ca746fe2b4..75af99931d 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
@@ -19,6 +19,7 @@ import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.RuleDefinition;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
@@ -39,10 +40,11 @@ import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
-import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -51,11 +53,34 @@ import java.util.Arrays;
import javax.annotation.Nullable;
/**
- * Parent class for repository-related Skyframe functions.
+ * Implementation of fetching various external repository types.
+ *
+ * <p>These objects are called from {@link RepositoryDelegatorFunction}.
+ *
+ * <p>External repositories come in two flavors: local and non-local.
+ *
+ * <p>Local ones are those whose fetching does not require access to any external resources
+ * (e.g. network). These are always re-fetched on Bazel server restarts. This operation is fast
+ * (usually just a few symlinks and maybe writing a BUILD file). {@code --nofetch} does not apply
+ * to local repositories.
+ *
+ * <p>The up-to-dateness of non-local repositories is checked using a marker file under the
+ * output base. When such a repository is fetched, data from the rule in the WORKSPACE file is
+ * written to the marker file which is consulted on next server startup. If the rule hasn't changed,
+ * the repository is not re-fetched.
+ *
+ * <p>Fetching repositories can be disabled using the {@code --nofetch} command line option. If a
+ * repository is on the file system, Bazel just tries to use it and hopes for the best. If the
+ * repository has never been fetched, Bazel errors out for lack of a better option. This is
+ * implemented using
+ * {@link com.google.devtools.build.lib.bazel.BazelRepositoryModule#REPOSITORY_VALUE_CHECKER} and
+ * a flag in {@link RepositoryValue} that tells Bazel whether the value in Skyframe is stale
+ * according to the value of {@code --nofetch} or not.
+ *
+ * <p>When a rule in the WORKSPACE file is changed, the corresponding {@link RepositoryValue} is
+ * invalidated using the usual Skyframe route.
*/
-public abstract class RepositoryFunction implements SkyFunction {
- protected static final byte[] NO_RULE_SPECIFIC_DATA = new byte[] {};
-
+public abstract class RepositoryFunction {
/**
* Exception thrown when something goes wrong accessing a remote repository.
*
@@ -85,13 +110,43 @@ public abstract class RepositoryFunction implements SkyFunction {
private BlazeDirectories directories;
private byte[] computeRuleKey(Rule rule, byte[] ruleSpecificData) {
-
return new Fingerprint()
.addBytes(RuleSerializer.serializeRule(rule).build().toByteArray())
.addBytes(ruleSpecificData)
.digestAndReset();
}
+ /**
+ * Fetch the remote repository represented by the given rule.
+ *
+ * <p>When this method is called, it has already been determined that the repository is stale and
+ * that it needs to be re-fetched.
+ */
+ @ThreadSafe
+ @Nullable
+ public abstract SkyValue fetch(Rule rule, Environment env)
+ throws SkyFunctionException, InterruptedException;
+
+ /**
+ * Whether fetching is done using local operations only.
+ *
+ * <p>If this is false, Bazel may decide not to re-fetch the repository, for example when the
+ * {@code --nofetch} command line option is used.
+ */
+ protected abstract boolean isLocal();
+
+ /**
+ * Returns a block of data that must be equal for two Rules for them to be considered the same.
+ *
+ * <p>This is used for the up-to-dateness check of fetched directory trees. The only reason for
+ * this to exist is the {@code maven_server} rule (which should go away, but until then, we need
+ * to keep it working somehow)
+ */
+ protected byte[] getRuleSpecificMarkerData(Rule rule, Environment env)
+ throws RepositoryFunctionException {
+ return new byte[] {};
+ }
+
private Path getMarkerPath(Rule rule) {
return getExternalRepositoryDirectory().getChild("@" + rule.getName() + ".marker");
}
@@ -103,7 +158,7 @@ public abstract class RepositoryFunction implements SkyFunction {
* <p>Deletes the marker file if not so that no matter what happens after, the state of the file
* system stays consistent.
*/
- protected boolean isFilesystemUpToDate(Rule rule, byte[] ruleSpecificData)
+ boolean isFilesystemUpToDate(Rule rule, byte[] ruleSpecificData)
throws RepositoryFunctionException {
try {
Path markerPath = getMarkerPath(rule);
@@ -126,7 +181,7 @@ public abstract class RepositoryFunction implements SkyFunction {
}
}
- protected void writeMarkerFile(Rule rule, byte[] ruleSpecificData)
+ void writeMarkerFile(Rule rule, byte[] ruleSpecificData)
throws RepositoryFunctionException {
try {
FileSystemUtils.writeContent(getMarkerPath(rule), computeRuleKey(rule, ruleSpecificData));
@@ -394,16 +449,6 @@ public abstract class RepositoryFunction implements SkyFunction {
return value;
}
- @Override
- public String extractTag(SkyKey skyKey) {
- return null;
- }
-
- /**
- * Gets Skyframe's name for this.
- */
- public abstract SkyFunctionName getSkyFunctionName();
-
/**
* Sets up output path information.
*/
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryValue.java
index 0b29f0ac93..ef9087144c 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryValue.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryValue.java
@@ -34,24 +34,35 @@ public class RepositoryValue implements SkyValue {
* it. Then user changes will be ignored (in favor of the cached version).
*/
private final Optional<FileValue> overlaidBuildFile;
+ private final boolean fetchingDelayed;
- private RepositoryValue(Path path, Optional<FileValue> overlaidBuildFile) {
+ private RepositoryValue(
+ Path path, Optional<FileValue> overlaidBuildFile, boolean fetchingDelayed) {
this.path = path;
this.overlaidBuildFile = overlaidBuildFile;
+ this.fetchingDelayed = fetchingDelayed;
}
/**
* Creates an immutable external repository.
*/
public static RepositoryValue create(Path repositoryDirectory) {
- return new RepositoryValue(repositoryDirectory, Optional.<FileValue>absent());
+ return new RepositoryValue(repositoryDirectory, Optional.<FileValue>absent(), false);
+ }
+
+ /**
+ * Creates a value that represents a repository whose fetching has been delayed by a
+ * {@code --nofetch} command line option.
+ */
+ public static RepositoryValue fetchingDelayed(Path repositoryDirectory) {
+ return new RepositoryValue(repositoryDirectory, Optional.<FileValue>absent(), true);
}
/**
* Creates an immutable external repository with a mutable BUILD file.
*/
public static RepositoryValue createNew(Path repositoryDirectory, FileValue overlaidBuildFile) {
- return new RepositoryValue(repositoryDirectory, Optional.of(overlaidBuildFile));
+ return new RepositoryValue(repositoryDirectory, Optional.of(overlaidBuildFile), false);
}
/**
@@ -68,6 +79,10 @@ public class RepositoryValue implements SkyValue {
return overlaidBuildFile;
}
+ public boolean isFetchingDelayed() {
+ return fetchingDelayed;
+ }
+
@Override
public boolean equals(Object other) {
if (this == other) {
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java
index 037862a0da..dbe8f37ff8 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java
@@ -77,11 +77,9 @@ public abstract class AnalysisMock {
ImmutableMap<String, RepositoryFunction> repositoryHandlers = ImmutableMap.of(
LocalRepositoryRule.NAME, localRepositoryFunction);
- return ImmutableMap.of(
+ return ImmutableMap.<SkyFunctionName, SkyFunction>of(
SkyFunctions.REPOSITORY,
- new RepositoryDelegatorFunction(directories, repositoryHandlers, new AtomicBoolean(true)),
- localRepositoryFunction.getSkyFunctionName(),
- localRepositoryFunction);
+ new RepositoryDelegatorFunction(directories, repositoryHandlers, new AtomicBoolean(true)));
}
public static class Delegate extends AnalysisMock {
diff --git a/src/test/shell/bazel/git_repository_test.sh b/src/test/shell/bazel/git_repository_test.sh
index 4beda0c47b..e9061d5b6c 100755
--- a/src/test/shell/bazel/git_repository_test.sh
+++ b/src/test/shell/bazel/git_repository_test.sh
@@ -267,6 +267,53 @@ EOF
}
+function test_git_repository_refetched_when_commit_changes() {
+ local repo_dir=$TEST_TMPDIR/repos/refetch
+
+ cd $WORKSPACE_DIR
+ cat > WORKSPACE <<EOF
+git_repository(name='g', remote='$repo_dir', commit='f0b79ff0')
+EOF
+
+ bazel build @g//:g >& $TEST_log || fail "Build failed"
+ expect_log "Cloning"
+ assert_contains "GIT 1" bazel-genfiles/external/g/go
+
+ cat > WORKSPACE <<EOF
+git_repository(name='g', remote='$repo_dir', commit='62777acc')
+EOF
+
+
+ bazel build @g//:g >& $TEST_log || fail "Build failed"
+ expect_log "Cloning"
+ assert_contains "GIT 2" bazel-genfiles/external/g/go
+}
+
+function test_git_repository_and_nofetch() {
+ local repo_dir=$TEST_TMPDIR/repos/refetch
+
+ cd $WORKSPACE_DIR
+ cat > WORKSPACE <<EOF
+git_repository(name='g', remote='$repo_dir', commit='f0b79ff0')
+EOF
+
+ bazel build --nofetch @g//:g >& $TEST_log && fail "Build succeeded"
+ expect_log "fetching repositories is disabled"
+ bazel build @g//:g >& $TEST_log || fail "Build failed"
+ assert_contains "GIT 1" bazel-genfiles/external/g/go
+
+ cat > WORKSPACE <<EOF
+git_repository(name='g', remote='$repo_dir', commit='62777acc')
+EOF
+
+
+ bazel build --nofetch @g//:g >& $TEST_log || fail "Build failed"
+ expect_log "External repository 'g' is not up-to-date"
+ assert_contains "GIT 1" bazel-genfiles/external/g/go
+ bazel build @g//:g >& $TEST_log || fail "Build failed"
+ assert_contains "GIT 2" bazel-genfiles/external/g/go
+}
+
# Helper function for setting up the workspace as follows
#
# $WORKSPACE_DIR/
diff --git a/src/test/shell/bazel/maven_test.sh b/src/test/shell/bazel/maven_test.sh
index 94683e6738..4f36ef61d5 100755
--- a/src/test/shell/bazel/maven_test.sh
+++ b/src/test/shell/bazel/maven_test.sh
@@ -213,7 +213,7 @@ EOF
chmod +x y.sh
bazel build //:y &> $TEST_log && fail "Building thing failed"
- expect_log "no such package '@x//'"
+ expect_log "does not represent an actual repository"
}
function test_auth() {