Replace UnixGlob's package glob prefetching functionality with an

optional method in FileSystem.  Custom FileSystem implementations can
use this to provide their own implementation of glob prefetching.

--
MOS_MIGRATED_REVID=140736304

4 files changed, 23 insertions, 44 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/packages/GlobCache.java b/src/main/java/com/google/devtools/build/lib/packages/GlobCache.java
index 5259385df3..56df73c58b 100644
--- a/src/main/java/com/google/devtools/build/lib/packages/GlobCache.java
+++ b/src/main/java/com/google/devtools/build/lib/packages/GlobCache.java
@@ -138,16 +138,8 @@ public class GlobCache {
     Future<List<Path>> cached = globCache.get(Pair.of(pattern, excludeDirs));
     if (cached == null) {
       if (maxDirectoriesToEagerlyVisit > -1
-          && !globalStarted.getAndSet(true)
-          && !pattern.startsWith("**")) {
-        UnixGlob.forPath(packageDirectory)
-            .setMaxDirectoriesToEagerlyVisit(maxDirectoriesToEagerlyVisit)
-            .addPattern("**")
-            .setExcludeDirectories(true)
-            .setDirectoryFilter(childDirectoryPredicate)
-            .setThreadPool(globExecutor)
-            .setFilesystemCalls(syscalls)
-            .globAsync(true);
+          && !globalStarted.getAndSet(true)) {
+        packageDirectory.prefetchPackageAsync(maxDirectoriesToEagerlyVisit);
       }
       cached = safeGlobUnsorted(pattern, excludeDirs);
       setGlobPaths(pattern, excludeDirs, cached);
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java
index 4d938049e0..de1fe986b7 100644
--- a/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java
+++ b/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java
@@ -827,4 +827,12 @@ public abstract class FileSystem {
    */
   protected abstract void createFSDependentHardLink(Path linkPath, Path originalPath)
       throws IOException;
+
+  /**
+   * Prefetch all directories and symlinks within the package
+   * rooted at "path".  Enter at most "maxDirs" total directories.
+   * Specializations for high-latency remote filesystems may wish to
+   * implement this in order to warm the filesystem's internal caches.
+   */
+  protected void prefetchPackageAsync(Path path, int maxDirs) { }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Path.java b/src/main/java/com/google/devtools/build/lib/vfs/Path.java
index e039efe74b..cdc0a0a996 100644
--- a/src/main/java/com/google/devtools/build/lib/vfs/Path.java
+++ b/src/main/java/com/google/devtools/build/lib/vfs/Path.java
@@ -1227,6 +1227,10 @@ public class Path implements Comparable<Path>, Serializable {
     fileSystem.chmod(this, mode);
   }
 
+  public void prefetchPackageAsync(int maxDirs) {
+    fileSystem.prefetchPackageAsync(this, maxDirs);
+  }
+
   /**
    * Compare Paths of the same file system using their PathFragments.
    *
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/UnixGlob.java b/src/main/java/com/google/devtools/build/lib/vfs/UnixGlob.java
index 82b5e553ce..9446aa33fd 100644
--- a/src/main/java/com/google/devtools/build/lib/vfs/UnixGlob.java
+++ b/src/main/java/com/google/devtools/build/lib/vfs/UnixGlob.java
@@ -44,7 +44,6 @@ import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Pattern;
@@ -71,7 +70,7 @@ public final class UnixGlob {
     GlobVisitor visitor =
         (threadPool == null)
             ? new GlobVisitor(checkForInterruption)
-            : new GlobVisitor(threadPool, checkForInterruption, -1);
+            : new GlobVisitor(threadPool, checkForInterruption);
     return visitor.glob(base, patterns, excludeDirectories, dirPred, syscalls);
   }
 
@@ -85,7 +84,7 @@ public final class UnixGlob {
     GlobVisitor visitor =
         (threadPool == null)
             ? new GlobVisitor(checkForInterruption)
-            : new GlobVisitor(threadPool, checkForInterruption, -1);
+            : new GlobVisitor(threadPool, checkForInterruption);
     visitor.glob(base, patterns, excludeDirectories, dirPred, syscalls);
     return visitor.getNumGlobTasksForTesting();
   }
@@ -97,10 +96,9 @@ public final class UnixGlob {
       Predicate<Path> dirPred,
       FilesystemCalls syscalls,
       boolean checkForInterruption,
-      ThreadPoolExecutor threadPool,
-      int maxDirectoriesToEagerlyVisit) {
+      ThreadPoolExecutor threadPool) {
     Preconditions.checkNotNull(threadPool, "%s %s", base, patterns);
-    return new GlobVisitor(threadPool, checkForInterruption, maxDirectoriesToEagerlyVisit)
+    return new GlobVisitor(threadPool, checkForInterruption)
         .globAsync(base, patterns, excludeDirectories, dirPred, syscalls);
   }
 
@@ -309,7 +307,6 @@ public final class UnixGlob {
     private ThreadPoolExecutor threadPool;
     private AtomicReference<? extends FilesystemCalls> syscalls =
         new AtomicReference<>(DEFAULT_SYSCALLS);
-    private int maxDirectoriesToEagerlyVisit = -1;
 
     /**
      * Creates a glob builder with the given base path.
@@ -390,11 +387,6 @@ public final class UnixGlob {
       return this;
     }
 
-    public Builder setMaxDirectoriesToEagerlyVisit(int maxDirectoriesToEagerlyVisit) {
-      this.maxDirectoriesToEagerlyVisit = maxDirectoriesToEagerlyVisit;
-      return this;
-    }
-
     /**
      * Executes the glob.
      */
@@ -439,8 +431,7 @@ public final class UnixGlob {
           pathFilter,
           syscalls.get(),
           checkForInterrupt,
-          threadPool,
-          maxDirectoriesToEagerlyVisit);
+          threadPool);
     }
   }
 
@@ -507,21 +498,17 @@ public final class UnixGlob {
     private final AtomicLong totalOps = new AtomicLong(0);
     private final AtomicLong pendingOps = new AtomicLong(0);
     private final AtomicReference<IOException> failure = new AtomicReference<>();
-    private final int maxDirectoriesToEagerlyVisit;
-    private final AtomicInteger visitedDirectories = new AtomicInteger(0);
     private volatile boolean canceled = false;
 
     GlobVisitor(
         ThreadPoolExecutor executor,
-        boolean failFastOnInterrupt,
-        int maxDirectoriesToEagerlyVisit) {
+        boolean failFastOnInterrupt) {
       this.executor = executor;
       this.result = new GlobFuture(this, failFastOnInterrupt);
-      this.maxDirectoriesToEagerlyVisit = maxDirectoriesToEagerlyVisit;
     }
 
     GlobVisitor(boolean failFastOnInterrupt) {
-      this(null, failFastOnInterrupt, -1);
+      this(null, failFastOnInterrupt);
     }
 
     /**
@@ -558,14 +545,6 @@ public final class UnixGlob {
     }
 
     /**
-     * Whether or not to store the results of this glob. If this glob is being done purely to warm
-     * the filesystem, we do not store the results, since it would take unnecessary memory.
-     */
-    private boolean storeGlobResults() {
-      return maxDirectoriesToEagerlyVisit == -1;
-    }
-
-    /**
      * Same as {@link #glob}, except does so asynchronously and returns a {@link Future} for the
      * result.
      */
@@ -780,7 +759,7 @@ public final class UnixGlob {
       }
 
       if (idx == context.patternParts.length) { // Base case.
-        if (storeGlobResults() && !(context.excludeDirectories && baseIsDir)) {
+        if (!(context.excludeDirectories && baseIsDir)) {
           results.add(base);
         }
 
@@ -792,10 +771,6 @@ public final class UnixGlob {
         return;
       }
 
-      if (maxDirectoriesToEagerlyVisit > -1
-          && visitedDirectories.incrementAndGet() > maxDirectoriesToEagerlyVisit) {
-        return;
-      }
       final String pattern = context.patternParts[idx];
 
       // ** is special: it can match nothing at all.
@@ -843,7 +818,7 @@ public final class UnixGlob {
             context.queueGlob(child, childIsDir, idx + 1);
           } else {
             // Instead of using an async call, just repeat the base case above.
-            if (storeGlobResults() && idx + 1 == context.patternParts.length) {
+            if (idx + 1 == context.patternParts.length) {
               results.add(child);
             }
           }