WorkspaceASTFunction returns a list of ASTs so we can split the AST before load statements

Issue #824 Step 2.

--
MOS_MIGRATED_REVID=113986176

6 files changed, 262 insertions, 22 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunction.java
index d1077d2260..46f5d0d9d1 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunction.java
@@ -14,9 +14,12 @@
 
 package com.google.devtools.build.lib.skyframe;
 
+import com.google.common.collect.ImmutableList;
 import com.google.devtools.build.lib.packages.RuleClassProvider;
 import com.google.devtools.build.lib.syntax.BuildFileAST;
+import com.google.devtools.build.lib.syntax.LoadStatement;
 import com.google.devtools.build.lib.syntax.ParserInputSource;
+import com.google.devtools.build.lib.syntax.Statement;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.devtools.build.lib.vfs.RootedPath;
@@ -27,6 +30,7 @@ import com.google.devtools.build.skyframe.SkyKey;
 import com.google.devtools.build.skyframe.SkyValue;
 
 import java.io.IOException;
+import java.util.List;
 
 /**
  * A SkyFunction to parse WORKSPACE files into a BuildFileAST.
@@ -65,12 +69,39 @@ public class WorkspaceASTFunction implements SkyFunction {
               new IOException("Failed to parse WORKSPACE file"), Transience.PERSISTENT);
         }
       }
-      return new WorkspaceASTValue(ast);
+      return new WorkspaceASTValue(splitAST(ast));
     } catch (IOException ex) {
       throw new WorkspaceASTFunctionException(ex, Transience.TRANSIENT);
     }
   }
 
+  /**
+   * Cut {@code ast} into a list of AST separated by load statements. We cut right before each load
+   * statement series.
+   */
+  private static ImmutableList<BuildFileAST> splitAST(BuildFileAST ast) {
+    ImmutableList.Builder<BuildFileAST> asts = ImmutableList.builder();
+    int prevIdx = 0;
+    boolean lastIsLoad = true; // don't cut if the first statement is a load.
+    List<Statement> statements = ast.getStatements();
+    for (int idx = 0; idx < statements.size(); idx++) {
+      Statement st = statements.get(idx);
+      if (st instanceof LoadStatement) {
+        if (!lastIsLoad) {
+          asts.add(ast.subTree(prevIdx, idx));
+          prevIdx = idx;
+        }
+        lastIsLoad = true;
+      } else {
+        lastIsLoad = false;
+      }
+    }
+    if (!statements.isEmpty()) {
+      asts.add(ast.subTree(prevIdx, statements.size()));
+    }
+    return asts.build();
+  }
+
   private static final class WorkspaceASTFunctionException extends SkyFunctionException {
     public WorkspaceASTFunctionException(Exception e, Transience transience) {
       super(e, transience);
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTValue.java
index 96f42dfa71..35479fca31 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTValue.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTValue.java
@@ -14,25 +14,46 @@
 package com.google.devtools.build.lib.skyframe;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
 import com.google.devtools.build.lib.syntax.BuildFileAST;
 import com.google.devtools.build.lib.vfs.RootedPath;
 import com.google.devtools.build.skyframe.SkyKey;
 import com.google.devtools.build.skyframe.SkyValue;
 
+import java.util.List;
+
 /**
- * A SkyValue that stores the parsed WORKSPACE file as an AST.
+ * A SkyValue that stores the parsed WORKSPACE file as a list of AST. Each AST contains the part
+ * of the WORKSPACE file between the first load statement of a series of load statements and the
+ * last statement before the next load statement. As example, the comment indicate where the next
+ * file would be split:
+ *
+ * <p><code>
+ * # First AST
+ * load('//foo:bar.bzl', 'foobar')
+ * foo_bar = 1
+ *
+ * # Second AST
+ * load('//foo:baz.bzl', 'foos')
+ * load('//bar:foo.bzl', 'bars')
+ * foos()
+ * bars()
+ *
+ * # Third AST
+ * load('//:bleh.bzl', 'bleh')
+ * </code>
  */
 public class WorkspaceASTValue implements SkyValue {
 
-  private final BuildFileAST ast;
+  private final ImmutableList<BuildFileAST> asts;
 
-  public WorkspaceASTValue(BuildFileAST ast) {
-    Preconditions.checkNotNull(ast);
-    this.ast = ast;
+  public WorkspaceASTValue(List<BuildFileAST> asts) {
+    Preconditions.checkNotNull(asts);
+    this.asts = ImmutableList.copyOf(asts);
   }
 
-  public BuildFileAST getAST() {
-    return ast;
+  public ImmutableList<BuildFileAST> getASTs() {
+    return asts;
   }
 
   public static SkyKey key(RootedPath path) {
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java
index fd948ee6f6..305d2e89fc 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java
@@ -54,9 +54,10 @@ public class WorkspaceFileFunction implements SkyFunction {
   public SkyValue compute(SkyKey skyKey, Environment env) throws WorkspaceFileFunctionException,
       InterruptedException {
 
-    RootedPath workspaceRoot = ((WorkspaceFileKey) skyKey.argument()).getPath();
+    WorkspaceFileKey key = (WorkspaceFileKey) skyKey.argument();
+    RootedPath workspaceRoot = key.getPath();
     WorkspaceASTValue workspaceASTValue =
-        (WorkspaceASTValue) env.getValue(new SkyKey(SkyFunctions.WORKSPACE_AST, workspaceRoot));
+        (WorkspaceASTValue) env.getValue(WorkspaceASTValue.key(workspaceRoot));
     if (workspaceASTValue == null) {
       return null;
     }
@@ -75,14 +76,15 @@ public class WorkspaceFileFunction implements SkyFunction {
               directories.getEmbeddedBinariesRoot(),
               directories.getWorkspace());
       try {
-        BuildFileAST ast = workspaceASTValue.getAST();
-        PackageFunction.SkylarkImportResult importResult =
-            PackageFunction.fetchImportsFromBuildFile(
-                repoWorkspace, Label.EXTERNAL_PACKAGE_IDENTIFIER, ast, env, null);
-        if (importResult != null) {
-          parser.execute(ast, importResult.importMap);
-        } else {
-          return null;
+        for (BuildFileAST ast : workspaceASTValue.getASTs()) {
+          PackageFunction.SkylarkImportResult importResult =
+              PackageFunction.fetchImportsFromBuildFile(
+                  repoWorkspace, Label.EXTERNAL_PACKAGE_IDENTIFIER, ast, env, null);
+          if (importResult != null) {
+            parser.execute(ast, importResult.importMap);
+          } else {
+            return null;
+          }
         }
       } catch (PackageFunctionException e) {
         throw new WorkspaceFileFunctionException(e, Transience.PERSISTENT);
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java b/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java
index 1ff08835a2..3d324b21bf 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java
@@ -59,6 +59,30 @@ public class BuildFileAST extends ASTNode {
     setLocation(result.location);
   }
 
+  private BuildFileAST(
+      ImmutableList<Statement> stmts,
+      boolean containsErrors,
+      String contentHashCode,
+      Location location) {
+    this.stmts = stmts;
+    this.containsErrors = containsErrors;
+    this.contentHashCode = contentHashCode;
+    this.comments = ImmutableList.of();
+    this.setLocation(location);
+  }
+
+  /**
+   * Extract a subtree containing only statements from {@code firstStatement} (included) up to
+   * {@code lastStatement} excluded.
+   */
+  public BuildFileAST subTree(int firstStatement, int lastStatement) {
+    return new BuildFileAST(
+        stmts.subList(firstStatement, lastStatement),
+        containsErrors,
+        null,
+        stmts.get(firstStatement).getLocation());
+  }
+
   /** Collects all load statements */
   private ImmutableList<SkylarkImport> fetchLoads(List<Statement> stmts) {
     ImmutableList.Builder<SkylarkImport> imports = new ImmutableList.Builder<>();
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunctionTest.java
new file mode 100644
index 0000000000..784c3e0707
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunctionTest.java
@@ -0,0 +1,162 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
+import com.google.devtools.build.lib.skyframe.WorkspaceFileFunctionTest.FakeFileValue;
+import com.google.devtools.build.lib.skyframe.WorkspaceFileFunctionTest.SkyKeyMatchers;
+import com.google.devtools.build.lib.syntax.BuildFileAST;
+import com.google.devtools.build.lib.testutil.TestRuleClassProvider;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Test for WorkspaceASTFunction.
+ */
+@RunWith(JUnit4.class)
+public class WorkspaceASTFunctionTest extends BuildViewTestCase {
+
+  private WorkspaceASTFunction astSkyFunc;
+  private FakeFileValue fakeWorkspaceFileValue;
+
+  @Before
+  public final void setUp() throws Exception {
+    ConfiguredRuleClassProvider ruleClassProvider = TestRuleClassProvider.getRuleClassProvider();
+    ConfiguredRuleClassProvider ruleClassProviderSpy = Mockito.spy(ruleClassProvider);
+    // Prevent returning default workspace file.
+    Mockito.when(ruleClassProviderSpy.getDefaultWorkspaceFile()).thenReturn("");
+    astSkyFunc = new WorkspaceASTFunction(ruleClassProviderSpy);
+    fakeWorkspaceFileValue = new FakeFileValue();
+  }
+
+  private RootedPath createWorkspaceFile(String... contents) throws IOException {
+    Path workspacePath = scratch.overwriteFile("WORKSPACE", contents);
+    fakeWorkspaceFileValue.setSize(workspacePath.getFileSize());
+    return RootedPath.toRootedPath(
+        workspacePath.getParentDirectory(), new PathFragment(workspacePath.getBaseName()));
+  }
+
+  private SkyFunction.Environment getEnv() {
+    SkyFunction.Environment env = Mockito.mock(SkyFunction.Environment.class);
+    Mockito.when(env.getValue(Matchers.argThat(new SkyKeyMatchers(SkyFunctions.FILE))))
+        .thenReturn(fakeWorkspaceFileValue);
+    return env;
+  }
+
+  private List<BuildFileAST> getASTs(String... lines)
+      throws IOException, SkyFunctionException, InterruptedException {
+    RootedPath workspacePath = createWorkspaceFile(lines);
+
+    WorkspaceASTValue value =
+        (WorkspaceASTValue) astSkyFunc.compute(WorkspaceASTValue.key(workspacePath), getEnv());
+    return value.getASTs();
+  }
+
+  @Test
+  public void testSplitASTNoLoad() throws IOException, SkyFunctionException, InterruptedException {
+    List<BuildFileAST> asts = getASTs("foo_bar = 1");
+    assertThat(asts).hasSize(1);
+    assertThat(asts.get(0).getStatements()).hasSize(1);
+  }
+
+  @Test
+  public void testSplitASTOneLoadAtTop()
+      throws IOException, SkyFunctionException, InterruptedException {
+    List<BuildFileAST> asts = getASTs("load('//:foo.bzl', 'bar')", "foo_bar = 1");
+    assertThat(asts).hasSize(1);
+    assertThat(asts.get(0).getStatements()).hasSize(2);
+  }
+
+  @Test
+  public void testSplitASTOneLoad() throws IOException, SkyFunctionException, InterruptedException {
+    List<BuildFileAST> asts = getASTs("foo_bar = 1", "load('//:foo.bzl', 'bar')");
+    assertThat(asts).hasSize(2);
+    assertThat(asts.get(0).getStatements()).hasSize(1);
+    assertThat(asts.get(1).getStatements()).hasSize(1);
+  }
+
+  @Test
+  public void testSplitASTTwoSuccessiveLoads()
+      throws IOException, SkyFunctionException, InterruptedException {
+    List<BuildFileAST> asts =
+        getASTs("foo_bar = 1", "load('//:foo.bzl', 'bar')", "load('//:bar.bzl', 'foo')");
+    assertThat(asts).hasSize(2);
+    assertThat(asts.get(0).getStatements()).hasSize(1);
+    assertThat(asts.get(1).getStatements()).hasSize(2);
+  }
+
+  @Test
+  public void testSplitASTTwoSucessiveLoadsWithNonLoadStatement()
+      throws IOException, SkyFunctionException, InterruptedException {
+    List<BuildFileAST> asts =
+        getASTs(
+            "foo_bar = 1",
+            "load('//:foo.bzl', 'bar')",
+            "load('//:bar.bzl', 'foo')",
+            "local_repository(name = 'foobar', path = '/bar/foo')");
+    assertThat(asts).hasSize(2);
+    assertThat(asts.get(0).getStatements()).hasSize(1);
+    assertThat(asts.get(1).getStatements()).hasSize(3);
+  }
+
+  @Test
+  public void testSplitASTThreeLoadsThreeSegments()
+      throws IOException, SkyFunctionException, InterruptedException {
+    List<BuildFileAST> asts =
+        getASTs(
+            "foo_bar = 1",
+            "load('//:foo.bzl', 'bar')",
+            "load('//:bar.bzl', 'foo')",
+            "local_repository(name = 'foobar', path = '/bar/foo')",
+            "load('@foobar//:baz.bzl', 'bleh')");
+    assertThat(asts).hasSize(3);
+    assertThat(asts.get(0).getStatements()).hasSize(1);
+    assertThat(asts.get(1).getStatements()).hasSize(3);
+    assertThat(asts.get(2).getStatements()).hasSize(1);
+  }
+
+  @Test
+  public void testSplitASTThreeLoadsThreeSegmentsWithContent()
+      throws IOException, SkyFunctionException, InterruptedException {
+    List<BuildFileAST> asts =
+        getASTs(
+            "foo_bar = 1",
+            "load('//:foo.bzl', 'bar')",
+            "load('//:bar.bzl', 'foo')",
+            "local_repository(name = 'foobar', path = '/bar/foo')",
+            "load('@foobar//:baz.bzl', 'bleh')",
+            "bleh()");
+    assertThat(asts).hasSize(3);
+    assertThat(asts.get(0).getStatements()).hasSize(1);
+    assertThat(asts.get(1).getStatements()).hasSize(3);
+    assertThat(asts.get(2).getStatements()).hasSize(2);
+  }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunctionTest.java
index 0f4ce405cf..208c3788c1 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunctionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunctionTest.java
@@ -60,7 +60,7 @@ public class WorkspaceFileFunctionTest extends BuildViewTestCase {
   private WorkspaceASTFunction astSkyFunc;
   private FakeFileValue fakeWorkspaceFileValue;
 
-  private static class FakeFileValue extends FileValue {
+  static class FakeFileValue extends FileValue {
     private boolean exists;
     private long size;
 
@@ -85,7 +85,7 @@ public class WorkspaceFileFunctionTest extends BuildViewTestCase {
       return exists;
     }
 
-    private void setExists(boolean exists) {
+    void setExists(boolean exists) {
       this.exists = exists;
     }
 
@@ -94,7 +94,7 @@ public class WorkspaceFileFunctionTest extends BuildViewTestCase {
       return size;
     }
 
-    private void setSize(long size) {
+    void setSize(long size) {
       this.size = size;
     }
   }
@@ -125,7 +125,7 @@ public class WorkspaceFileFunctionTest extends BuildViewTestCase {
   }
 
   // Dummy harmcrest matcher that match the function name of a skykey
-  private static class SkyKeyMatchers extends BaseMatcher<SkyKey> {
+  static class SkyKeyMatchers extends BaseMatcher<SkyKey> {
     private final SkyFunctionName functionName;
     
     public SkyKeyMatchers(SkyFunctionName functionName) {