Add a `sync` command

Add a command that ensures that all skylark workspace rules get called.
In follow-up changes the semantics of the sync command will be extended
to call all those rules unconditionally. This, together with recoding
of the return values of the repository rules, as currently provided by
--experimental_repository_resolved_file provides the framework for
resolving underspecified rules, e.g., rules following head of an external
repository.

Change-Id: I11061ec138a9ba7a7b61a431eeb1b8667dfabb95
PiperOrigin-RevId: 199792026

5 files changed, 141 insertions, 3 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD
index fbfcff804b..3e3c1b7984 100644
--- a/src/main/java/com/google/devtools/build/lib/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/BUILD
@@ -757,7 +757,10 @@ java_library(
 java_library(
     name = "bazel-commands",
     srcs = glob(["bazel/commands/*.java"]),
-    resources = ["bazel/commands/fetch.txt"],
+    resources = [
+        "bazel/commands/fetch.txt",
+        "bazel/commands/sync.txt",
+    ],
     deps = [
         ":keep-going-option",
         "//src/main/java/com/google/devtools/build/lib:build-base",
@@ -771,6 +774,10 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/query2",
         "//src/main/java/com/google/devtools/build/lib/query2:abstract-blaze-query-env",
         "//src/main/java/com/google/devtools/build/lib/query2:query-engine",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
+        "//src/main/java/com/google/devtools/build/skyframe",
+        "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
         "//src/main/java/com/google/devtools/common/options",
         "//third_party:guava",
     ],
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
index 58441b27fb..742f35e13d 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -24,6 +24,7 @@ import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
 import com.google.devtools.build.lib.analysis.RuleDefinition;
 import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
 import com.google.devtools.build.lib.bazel.commands.FetchCommand;
+import com.google.devtools.build.lib.bazel.commands.SyncCommand;
 import com.google.devtools.build.lib.bazel.repository.GitRepositoryFunction;
 import com.google.devtools.build.lib.bazel.repository.HttpArchiveFunction;
 import com.google.devtools.build.lib.bazel.repository.HttpFileFunction;
@@ -181,6 +182,7 @@ public class BazelRepositoryModule extends BlazeModule {
   @Override
   public void serverInit(OptionsProvider startupOptions, ServerBuilder builder) {
     builder.addCommands(new FetchCommand());
+    builder.addCommands(new SyncCommand());
     builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache));
   }
 
@@ -295,7 +297,7 @@ public class BazelRepositoryModule extends BlazeModule {
 
   @Override
   public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
-    return ImmutableSet.of("fetch", "build", "query").contains(command.name())
+    return ImmutableSet.of("sync", "fetch", "build", "query").contains(command.name())
         ? ImmutableList.<Class<? extends OptionsBase>>of(RepositoryOptions.class)
         : ImmutableList.<Class<? extends OptionsBase>>of();
   }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java
new file mode 100644
index 0000000000..c5c30852fa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java
@@ -0,0 +1,119 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.commands;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
+import com.google.devtools.build.lib.cmdline.RepositoryName;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandResult;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.build.lib.runtime.KeepGoingOption;
+import com.google.devtools.build.lib.skyframe.PackageLookupValue;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import com.google.devtools.build.lib.skyframe.WorkspaceFileValue;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+/** Syncs all repositories specifed in the workspace file */
+@Command(
+    name = SyncCommand.NAME,
+    options = {PackageCacheOptions.class, KeepGoingOption.class},
+    help = "resource:sync.txt",
+    shortDescription = "Syncs all repositories specifed in the workspace file",
+    allowResidue = false)
+public final class SyncCommand implements BlazeCommand {
+  public static final String NAME = "sync";
+
+  @Override
+  public void editOptions(OptionsParser optionsParser) {}
+
+  @Override
+  public BlazeCommandResult exec(CommandEnvironment env, OptionsProvider options) {
+    try {
+      env.setupPackageCache(options, env.getRuntime().getDefaultsPackageContent());
+      SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor();
+
+      // Obtain the key for the top-level WORKSPACE file
+      SkyKey packageLookupKey = PackageLookupValue.key(Label.EXTERNAL_PACKAGE_IDENTIFIER);
+      EvaluationResult<SkyValue> packageLookupValue =
+          skyframeExecutor.prepareAndGet(
+              ImmutableSet.of(packageLookupKey),
+              SkyframeExecutor.DEFAULT_THREAD_COUNT,
+              env.getReporter());
+      if (packageLookupValue.hasError()) {
+        return BlazeCommandResult.exitCode(ExitCode.ANALYSIS_FAILURE);
+      }
+      RootedPath workspacePath =
+          ((PackageLookupValue) packageLookupValue.get(packageLookupKey))
+              .getRootedPath(Label.EXTERNAL_PACKAGE_IDENTIFIER);
+      SkyKey workspace = WorkspaceFileValue.key(workspacePath);
+
+      // read and evaluate the WORKSPACE file to its end
+      WorkspaceFileValue fileValue = null;
+      while (workspace != null) {
+        EvaluationResult<SkyValue> value =
+            skyframeExecutor.prepareAndGet(
+                ImmutableSet.of(workspace),
+                SkyframeExecutor.DEFAULT_THREAD_COUNT,
+                env.getReporter());
+        if (value.hasError()) {
+          return BlazeCommandResult.exitCode(ExitCode.ANALYSIS_FAILURE);
+        }
+        fileValue = (WorkspaceFileValue) value.get(workspace);
+        workspace = fileValue.next();
+      }
+
+      // take all skylark workspace rules and get their values
+      ImmutableSet.Builder<SkyKey> repositoriesToFetch = new ImmutableSet.Builder<>();
+      for (Rule rule : fileValue.getPackage().getTargets(Rule.class)) {
+        if (rule.getRuleClassObject().getWorkspaceOnly() && rule.getRuleClassObject().isSkylark()) {
+          // TODO(aehlig): avoid the detour of serializing and then parsing the repository name
+          try {
+            repositoriesToFetch.add(
+                RepositoryDirectoryValue.key(RepositoryName.create("@" + rule.getName())));
+          } catch (LabelSyntaxException e) {
+            return BlazeCommandResult.exitCode(ExitCode.BLAZE_INTERNAL_ERROR);
+          }
+        }
+      }
+      EvaluationResult<SkyValue> fetchValue;
+      fetchValue =
+          skyframeExecutor.prepareAndGet(
+              repositoriesToFetch.build(),
+              SkyframeExecutor.DEFAULT_THREAD_COUNT,
+              env.getReporter());
+      if (fetchValue.hasError()) {
+        return BlazeCommandResult.exitCode(ExitCode.ANALYSIS_FAILURE);
+      }
+    } catch (InterruptedException e) {
+      return BlazeCommandResult.exitCode(ExitCode.INTERRUPTED);
+    } catch (AbruptExitException e) {
+      return BlazeCommandResult.exitCode(ExitCode.LOCAL_ENVIRONMENTAL_ERROR);
+    }
+
+    return BlazeCommandResult.exitCode(ExitCode.SUCCESS);
+  }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/sync.txt b/src/main/java/com/google/devtools/build/lib/bazel/commands/sync.txt
new file mode 100644
index 0000000000..0e651c5600
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/sync.txt
@@ -0,0 +1,10 @@
+
+Usage: %{product} %{command} [<option> ...]
+
+Ensures that all Skylark repository rules of the top-level WORKSPACE
+file are called.
+
+NOTE: This command is still very experimental and the precise semantics
+will change in the near future.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java
index 1664957a03..d4d4849e6c 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java
@@ -39,7 +39,7 @@ public final class RepositoryResolvedModule extends BlazeModule {
 
   @Override
   public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
-    return ImmutableSet.of("fetch", "build", "query").contains(command.name())
+    return ImmutableSet.of("sync", "fetch", "build", "query").contains(command.name())
         ? ImmutableList.<Class<? extends OptionsBase>>of(RepositoryResolvedOptions.class)
         : ImmutableList.<Class<? extends OptionsBase>>of();
   }