diff options
10 files changed, 146 insertions, 23 deletions
diff --git a/bootstrap_test.sh b/bootstrap_test.sh index 1ed31f7dff..3560187b29 100755 --- a/bootstrap_test.sh +++ b/bootstrap_test.sh @@ -75,6 +75,7 @@ function bootstrap() { local BAZEL_SUM=$2 [ -x "${BAZEL_BIN}" ] || fail "syntax: bootstrap bazel-binary" ${BAZEL_BIN} --blazerc=/dev/null clean || return $? + ${BAZEL_BIN} --blazerc=/dev/null fetch //... || return $? ${BAZEL_BIN} --blazerc=/dev/null build --nostamp //src:bazel //src:tools || return $? if [ -n "${BAZEL_SUM}" ]; then diff --git a/compile.sh b/compile.sh index cb5c23add2..48e50699e9 100755 --- a/compile.sh +++ b/compile.sh @@ -500,4 +500,7 @@ else [[ $package_path != $old_line ]] && log "$warning" fi +# Run "bazel fetch" to bring in the JDK (so users don't have to). +${PWD}/output/bazel fetch //... + log "Build successful! Binary is here: ${PWD}/output/bazel" diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index fa8cc57033..a4b6910894 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -39,6 +39,8 @@ import com.google.devtools.build.lib.bazel.rules.workspace.NewHttpArchiveRule; import com.google.devtools.build.lib.bazel.rules.workspace.NewLocalRepositoryRule; import com.google.devtools.build.lib.runtime.BlazeCommand; import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.util.Clock; import com.google.devtools.build.lib.vfs.Path; @@ -49,6 +51,7 @@ import com.google.devtools.common.options.OptionsProvider; import java.util.Map.Entry; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; /** * Adds support for fetching external code. @@ -58,6 +61,7 @@ public class BazelRepositoryModule extends BlazeModule { private BlazeDirectories directories; // A map of repository handlers that can be looked up by rule class name. private final ImmutableMap<String, RepositoryFunction> repositoryHandlers; + private final AtomicBoolean isFetch = new AtomicBoolean(false); public BazelRepositoryModule() { repositoryHandlers = ImmutableMap.<String, RepositoryFunction>builder() @@ -104,6 +108,11 @@ public class BazelRepositoryModule extends BlazeModule { } @Override + public void beforeCommand(BlazeRuntime blazeRuntime, Command command) { + isFetch.set(command.name().equals(FetchCommand.NAME)); + } + + @Override public ImmutableMap<SkyFunctionName, SkyFunction> getSkyFunctions(BlazeDirectories directories) { ImmutableMap.Builder<SkyFunctionName, SkyFunction> builder = ImmutableMap.builder(); @@ -114,7 +123,7 @@ public class BazelRepositoryModule extends BlazeModule { // Create the delegator everything flows through. builder.put(SkyFunctions.REPOSITORY, - new RepositoryDelegatorFunction(repositoryHandlers)); + new RepositoryDelegatorFunction(directories, repositoryHandlers, isFetch)); // Helper SkyFunctions. builder.put(SkyFunctionName.computed(HttpDownloadFunction.NAME), new HttpDownloadFunction()); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java index 3bdbc95248..c26d7ad756 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.bazel.commands; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.devtools.build.lib.Constants; @@ -24,6 +25,7 @@ import com.google.devtools.build.lib.query2.AbstractBlazeQueryEnvironment; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Setting; import com.google.devtools.build.lib.query2.engine.QueryException; import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.rules.java.JavaOptions; import com.google.devtools.build.lib.runtime.BlazeCommand; import com.google.devtools.build.lib.runtime.BlazeRuntime; import com.google.devtools.build.lib.runtime.Command; @@ -36,8 +38,12 @@ import com.google.devtools.common.options.OptionsProvider; /** * Fetches external repositories. Which is so fetch. */ -@Command(name = "fetch", - options = { PackageCacheOptions.class }, +@Command(name = FetchCommand.NAME, + options = { + PackageCacheOptions.class, + FetchOptions.class, + JavaOptions.class, + }, help = "resource:fetch.txt", shortDescription = "Fetches external repositories that are prerequisites to the targets.", allowResidue = true, @@ -46,7 +52,8 @@ public final class FetchCommand implements BlazeCommand { // TODO(kchodorow): add an option to force-fetch targets, even if they're already downloaded. // TODO(kchodorow): this would be a great time to check for difference and invalidate the upward // transitive closure for local repositories. - // TODO(kchodorow): prevent fetching from being done during a build. + + public static final String NAME = "fetch"; @Override public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { } @@ -73,16 +80,25 @@ public final class FetchCommand implements BlazeCommand { } // Querying for all of the dependencies of the targets has the side-effect of populating the - // Skyframe graph for external targets, which requires downloading them. - String query = Joiner.on(" union ").join(options.getResidue()); + // Skyframe graph for external targets, which requires downloading them. The JDK is required to + // build everything but isn't counted as a dep in the build graph so we add it manually. + JavaOptions javaOptions = options.getOptions(JavaOptions.class); + ImmutableList.Builder<String> labelsToLoad = new ImmutableList.Builder<String>() + .addAll(options.getResidue()); + if (String.valueOf(javaOptions.javaLangtoolsJar).equals(JavaOptions.DEFAULT_LANGTOOLS)) { + labelsToLoad.add(javaOptions.javaBase); + } else { + // TODO(kchodroow): Remove this when OS X isn't as hacky about finding the JVM. Our test + // framework currently doesn't set up the JDK normally on OS X, so attempting to fetch + // tools/jdk:jdk will cause errors. + labelsToLoad.add(String.valueOf(javaOptions.javaToolchain)); + } + String query = Joiner.on(" union ").join(labelsToLoad.build()); query = "deps(" + query + ")"; AbstractBlazeQueryEnvironment<Target> env = QueryCommand.newQueryEnvironment( - runtime, - true, - false, - Lists.<String>newArrayList(), 4, - Sets.<Setting>newHashSet()); + runtime, options.getOptions(FetchOptions.class).keepGoing, false, + Lists.<String>newArrayList(), 200, Sets.<Setting>newHashSet()); // 1. Parse query: QueryExpression expr; @@ -102,6 +118,9 @@ public final class FetchCommand implements BlazeCommand { runtime.getReporter().handle(Event.error(e.getMessage())); return ExitCode.COMMAND_LINE_ERROR; } + + runtime.getReporter().handle( + Event.progress("All external dependencies fetched successfully.")); return ExitCode.SUCCESS; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchOptions.java new file mode 100644 index 0000000000..c0f970e73e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchOptions.java @@ -0,0 +1,33 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.commands; + +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +/** + * Command-line options for the fetch command. + */ +public class FetchOptions extends OptionsBase { + @Option(name = "keep_going", + abbrev = 'k', + defaultValue = "false", + category = "strategy", + help = "Continue as much as possible after an error. While the " + + "target that failed and those that depend on it cannot be " + + "analyzed, other prerequisites of these " + + "targets can be.") + public boolean keepGoing; +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryDelegatorFunction.java index f0af0c64ec..514884f809 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryDelegatorFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryDelegatorFunction.java @@ -15,10 +15,15 @@ package com.google.devtools.build.lib.bazel.repository; import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.bazel.repository.RepositoryFunction.RepositoryFunctionException; import com.google.devtools.build.lib.packages.NoSuchPackageException; import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName; import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.skyframe.FileValue; +import com.google.devtools.build.lib.skyframe.RepositoryValue; import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; @@ -26,6 +31,7 @@ import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; /** * Implements delegation to the correct repository fetcher. @@ -35,9 +41,17 @@ public class RepositoryDelegatorFunction implements SkyFunction { // Mapping of rule class name to SkyFunction. private final ImmutableMap<String, RepositoryFunction> handlers; + // This is a reference to isFetch in BazelRepositoryModule, which tracks whether the current + // command is a fetch. Remote repository lookups are only allowed during fetches. + private final AtomicBoolean isFetch; + private final BlazeDirectories directories; + public RepositoryDelegatorFunction( - ImmutableMap<String, RepositoryFunction> handlers) { + BlazeDirectories directories, ImmutableMap<String, RepositoryFunction> handlers, + AtomicBoolean isFetch) { + this.directories = directories; this.handlers = handlers; + this.isFetch = isFetch; } @Override @@ -47,6 +61,27 @@ public class RepositoryDelegatorFunction implements SkyFunction { if (rule == null) { return null; } + + // If Bazel isn't running a fetch command, we shouldn't be able to download anything. To + // prevent having to rerun fetch on server restart, we check if the external repository + // directory already exists and, if it does, just use that. + if (!isFetch.get()) { + FileValue repoRoot = RepositoryFunction.getRepositoryDirectory( + RepositoryFunction.getExternalRepositoryDirectory(directories) + .getRelative(rule.getName()), env); + if (repoRoot == null) { + return null; + } + Path repoPath = repoRoot.realRootedPath().asPath(); + if (!repoPath.exists()) { + throw new RepositoryFunctionException(new IOException( + "to fix, run\n\tbazel fetch //...\nExternal repository " + repositoryName + + " not found"), + Transience.TRANSIENT); + } + return RepositoryValue.create(repoRoot); + } + RepositoryFunction handler = handlers.get(rule.getRuleClass()); if (handler == null) { throw new IllegalStateException("Could not find handler for " + rule); @@ -57,11 +92,11 @@ public class RepositoryDelegatorFunction implements SkyFunction { return env.getValueOrThrow( key, NoSuchPackageException.class, IOException.class, EvalException.class); } catch (NoSuchPackageException e) { - throw new RepositoryFunction.RepositoryFunctionException(e, Transience.PERSISTENT); + throw new RepositoryFunctionException(e, Transience.PERSISTENT); } catch (IOException e) { - throw new RepositoryFunction.RepositoryFunctionException(e, Transience.PERSISTENT); + throw new RepositoryFunctionException(e, Transience.PERSISTENT); } catch (EvalException e) { - throw new RepositoryFunction.RepositoryFunctionException(e, Transience.PERSISTENT); + throw new RepositoryFunctionException(e, Transience.PERSISTENT); } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java index dbabc47832..f2fad7f1f2 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java @@ -41,8 +41,8 @@ import java.util.Set; */ public class JavaOptions extends FragmentOptions { // Defaults value for options + public static final String DEFAULT_LANGTOOLS = "//tools/jdk:langtools"; static final String DEFAULT_LANGTOOLS_BOOTCLASSPATH = "//tools/jdk:bootclasspath"; - static final String DEFAULT_LANGTOOLS = "//tools/jdk:langtools"; static final String DEFAULT_JAVABUILDER = "//tools/jdk:JavaBuilder_deploy.jar"; static final String DEFAULT_SINGLEJAR = "//tools/jdk:SingleJar_deploy.jar"; static final String DEFAULT_JAVABASE = "//tools/jdk:jdk"; diff --git a/src/test/shell/bazel/external_integration_test.sh b/src/test/shell/bazel/external_integration_test.sh index f3b201e7b5..6e24c013b6 100755 --- a/src/test/shell/bazel/external_integration_test.sh +++ b/src/test/shell/bazel/external_integration_test.sh @@ -169,6 +169,7 @@ cat external/endangered/fox/male EOF chmod +x zoo/female.sh + bazel fetch //zoo:breeding-program || fail "Fetch failed" bazel run //zoo:breeding-program >& $TEST_log \ || echo "Expected build/run to succeed" kill_nc @@ -197,8 +198,7 @@ cat fox/male EOF chmod +x zoo/female.sh - bazel run //zoo:breeding-program >& $TEST_log && echo "Expected build to fail" - cat $TEST_log + bazel fetch //zoo:breeding-program >& $TEST_log && fail "Expected fetch to fail" expect_log "Connection refused" } @@ -235,7 +235,7 @@ cat fox/male EOF chmod +x zoo/female.sh - bazel run //zoo:breeding-program >& $TEST_log && echo "Expected build to fail" + bazel fetch //zoo:breeding-program >& $TEST_log && echo "Expected fetch to fail" kill_nc expect_log "does not match expected SHA-256" } @@ -262,6 +262,7 @@ EOF nc_l $nc_port < $http_response >& $nc_log & pid=$! + bazel fetch //zoo:breeding-program || fail "Fetch failed" bazel run //zoo:breeding-program >& $TEST_log \ || echo "Expected run to succeed" kill_nc @@ -298,6 +299,7 @@ public class BallPit { } EOF + bazel fetch //zoo:ball-pit || fail "Fetch failed" bazel run //zoo:ball-pit >& $TEST_log || echo "Expected run to succeed" kill_nc expect_log "Tra-la!" @@ -309,7 +311,7 @@ function test_invalid_rule() { http_jar(name = 'endangered', sha256 = 'dummy') EOF - bazel run //external:endangered >& $TEST_log && echo "Expected run to fail" + bazel fetch //external:endangered >& $TEST_log && fail "Expected fetch to fail" expect_log "missing value for mandatory attribute 'url' in 'http_jar' rule" } @@ -327,7 +329,8 @@ maven_jar( bind(name = 'mongoose', actual = '@endangered//jar') EOF - bazel run //zoo:ball-pit >& $TEST_log || echo "Expected run to succeed" + bazel fetch //zoo:ball-pit || fail "Fetch failed" + bazel run //zoo:ball-pit >& $TEST_log || fail "Expected run to succeed" kill_nc assert_contains "GET /com/example/carnivore/carnivore/1.23/carnivore-1.23.jar" $nc_log expect_log "Tra-la!" @@ -354,7 +357,7 @@ maven_jar( bind(name = 'mongoose', actual = '@endangered//jar') EOF - bazel run //zoo:ball-pit >& $TEST_log && echo "Expected run to fail" + bazel fetch //zoo:ball-pit >& $TEST_log && echo "Expected fetch to fail" kill_nc expect_log "Failed to fetch Maven dependency: Could not find artifact" } @@ -408,6 +411,7 @@ EOF chmod +x zoo/female.sh bazel clean --expunge + bazel fetch //zoo:breeding-program || fail "Fetch failed" bazel run //zoo:breeding-program >& $TEST_log \ || echo "Expected build/run to succeed" kill_nc @@ -438,6 +442,15 @@ EOF # Rerun fetch while nc isn't serving anything to make sure the fetched result # is cached. bazel fetch //zoo:ball-pit >& $TEST_log || fail "Incremental fetch failed" + + # Make sure fetch isn't needed after a bazel restart. + bazel shutdown + bazel build //zoo:ball-pit >& $TEST_log || fail "Fetch shouldn't be required" + + # But it is required after a clean. + bazel clean --expunge + bazel build //zoo:ball-pit >& $TEST_log && fail "Expected build to fail" + expect_log "bazel fetch //..." } run_suite "external tests" diff --git a/src/test/shell/bazel/local_repository_test.sh b/src/test/shell/bazel/local_repository_test.sh index 7cf7371456..60a95100f8 100755 --- a/src/test/shell/bazel/local_repository_test.sh +++ b/src/test/shell/bazel/local_repository_test.sh @@ -85,7 +85,7 @@ EOF echo "feed bamboo" > red/day-keeper - + bazel fetch //zoo:dumper || fail "Fetch failed" bazel run //zoo:dumper >& $TEST_log || fail "Failed to build/run zoo" expect_log "rawr" "//external runfile not cat-ed" expect_log "feed bamboo" \ @@ -140,6 +140,7 @@ public class BallPit { } EOF + bazel fetch //zoo:ball-pit || fail "Fetch failed" bazel run //zoo:ball-pit >& $TEST_log expect_log "Tra-la!" } @@ -207,6 +208,7 @@ java_library( visibility = ["//visibility:public"], ) EOF + bazel fetch //zoo:ball-pit || fail "Fetch failed" bazel run //zoo:ball-pit >& $TEST_log || fail "Failed to build/run zoo" expect_log "Tra-la!" @@ -222,6 +224,7 @@ EOF # Check that rebuilding this doesn't rebuild libmongoose.jar, even though it # has changed. Bazel assumes that files in external repositories are # immutable. + bazel fetch //zoo:ball-pit || fail "Fetch failed" bazel run //zoo:ball-pit >& $TEST_log || fail "Failed to build/run zoo" expect_log "Tra-la!" expect_not_log "Building endangered/libmongoose.jar" @@ -229,6 +232,7 @@ EOF } function test_default_ws() { + bazel fetch //external:java || fail "Fetch failed" bazel build //external:java >& $TEST_log || fail "Failed to build java" } @@ -283,6 +287,7 @@ bind( ) EOF + bazel fetch //:greeter || fail "Fetch failed" bazel run //:greeter >& $TEST_log || fail "Failed to run greeter" expect_log "Hello" } @@ -361,6 +366,7 @@ bind( ) EOF + bazel fetch //a:a || fail "Fetch failed" bazel build //a:a >& $TEST_log && fail "Building //a:a should error out" expect_log "** Please add the following dependencies:" expect_log "@x-repo//x to //a:a" @@ -417,6 +423,7 @@ int main() { } EOF + bazel fetch //:printer || fail "Fetch failed" bazel run //:printer >& $TEST_log || fail "Running //:printer failed" expect_log "My number is 3" } @@ -431,6 +438,7 @@ local_repository( path = "$external_dir", ) EOF + bazel fetch //external:* || fail "Fetch failed" bazel query 'deps(//external:*)' >& $TEST_log || fail "query failed" expect_log "//external:my-repo" } @@ -459,6 +467,7 @@ genrule( visibility = ["//visibility:public"], ) EOF + bazel fetch //external:best-turtle || fail "Fetch failed" bazel build //external:best-turtle &> $TEST_log || fail "First build failed" assert_contains "Raphael" bazel-genfiles/tmnt diff --git a/src/test/shell/bazel/test-setup.sh b/src/test/shell/bazel/test-setup.sh index b26656c83a..7e83685a83 100755 --- a/src/test/shell/bazel/test-setup.sh +++ b/src/test/shell/bazel/test-setup.sh @@ -205,3 +205,4 @@ function assert_bazel_run() { setup_bazelrc setup_clean_workspace +bazel fetch //tools/jdk/... |