diff options
Diffstat (limited to 'src/main')
6 files changed, 547 insertions, 222 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD index 19d181f6bf..419adffe0f 100644 --- a/src/main/java/com/google/devtools/build/lib/BUILD +++ b/src/main/java/com/google/devtools/build/lib/BUILD @@ -485,6 +485,7 @@ java_library( ":shell", ":skylarkinterface", ":transitive-info-provider", + ":unix", ":util", ":vfs", "//src/main/java/com/google/devtools/build/lib/actions", diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/LocalDiffAwareness.java b/src/main/java/com/google/devtools/build/lib/skyframe/LocalDiffAwareness.java index 4a15baad34..cae5cdebd4 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/LocalDiffAwareness.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/LocalDiffAwareness.java @@ -15,9 +15,7 @@ package com.google.devtools.build.lib.skyframe; import com.google.common.base.Function; -import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.util.Preconditions; @@ -25,28 +23,21 @@ import com.google.devtools.build.lib.vfs.ModifiedFileSet; import com.google.devtools.build.lib.vfs.PathFragment; import java.io.IOException; -import java.nio.file.ClosedWatchServiceException; import java.nio.file.FileSystems; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.LinkOption; import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.StandardWatchEventKinds; -import java.nio.file.WatchEvent; -import java.nio.file.WatchEvent.Kind; -import java.nio.file.WatchKey; import java.nio.file.WatchService; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.HashSet; import java.util.Set; /** - * File system watcher for local filesystems. It's able to provide a list of changed - * files between two consecutive calls. Uses the standard Java WatchService, which uses - * 'inotify' on Linux. + * File system watcher for local filesystems. It's able to provide a list of changed files between + * two consecutive calls. On Linux, uses the standard Java WatchService, which uses 'inotify' and, + * on OS X, uses {@link MacOSXFsEventsDiffAwareness}, which use FSEvents. + * + * <p> + * This is an abstract class, specialized by {@link MacOSXFsEventsDiffAwareness} and + * {@link WatchServiceDiffAwareness}. */ -public class LocalDiffAwareness implements DiffAwareness { +public abstract class LocalDiffAwareness implements DiffAwareness { /** Factory for creating {@link LocalDiffAwareness} instances. */ public static class Factory implements DiffAwareness.Factory { @@ -77,9 +68,9 @@ public class LocalDiffAwareness implements DiffAwareness { return null; } } - // Disable the file watcher on OSX due to https://bugs.openjdk.java.net/browse/JDK-7133447 + // On OSX uses FsEvents due to https://bugs.openjdk.java.net/browse/JDK-7133447 if (OS.getCurrent() == OS.DARWIN) { - return null; + return new MacOSXFsEventsDiffAwareness(resolvedPathEntryFragment.toString()); } WatchService watchService; @@ -88,28 +79,17 @@ public class LocalDiffAwareness implements DiffAwareness { } catch (IOException e) { return null; } - return new LocalDiffAwareness(resolvedPathEntryFragment.toString(), - watchService); + return new WatchServiceDiffAwareness(resolvedPathEntryFragment.toString(), watchService); } } private int numGetCurrentViewCalls = 0; - /** - * Bijection from WatchKey to the (absolute) Path being watched. WatchKeys don't have this - * functionality built-in so we do it ourselves. - */ - private final HashBiMap<WatchKey, Path> watchKeyToDirBiMap = HashBiMap.create(); - /** Root directory to watch. This is an absolute path. */ - private final Path watchRootPath; + protected final Path watchRootPath; - /** Every directory is registered under this watch service. */ - private WatchService watchService; - - private LocalDiffAwareness(String watchRoot, WatchService watchService) { + protected LocalDiffAwareness(String watchRoot) { this.watchRootPath = FileSystems.getDefault().getPath(watchRoot); - this.watchService = watchService; } /** @@ -138,33 +118,20 @@ public class LocalDiffAwareness implements DiffAwareness { } } - @Override - public SequentialView getCurrentView() throws BrokenDiffAwarenessException { - Set<Path> modifiedAbsolutePaths; - if (numGetCurrentViewCalls++ == 0) { - try { - registerSubDirectoriesAndReturnContents(watchRootPath); - } catch (IOException e) { - close(); - throw new BrokenDiffAwarenessException( - "Error encountered with local file system watcher " + e); - } - modifiedAbsolutePaths = ImmutableSet.of(); - } else { - try { - modifiedAbsolutePaths = collectChanges(); - } catch (BrokenDiffAwarenessException e) { - close(); - throw e; - } catch (IOException e) { - close(); - throw new BrokenDiffAwarenessException( - "Error encountered with local file system watcher " + e); - } catch (ClosedWatchServiceException e) { - throw new BrokenDiffAwarenessException( - "Internal error with the local file system watcher " + e); - } - } + /** + * Returns true on any call before first call to {@link #newView(Set<Path>)}. + */ + protected boolean isFirstCall() { + return numGetCurrentViewCalls == 0; + } + + /** + * Create a new views using a list of modified absolute paths. This will increase the view + * counter. + */ + protected SequentialView newView(Set<Path> modifiedAbsolutePaths) + throws BrokenDiffAwarenessException { + numGetCurrentViewCalls++; return new SequentialView(this, numGetCurrentViewCalls, modifiedAbsolutePaths); } @@ -193,168 +160,14 @@ public class LocalDiffAwareness implements DiffAwareness { return "local"; } - @Override - public void close() { - try { - watchService.close(); - } catch (IOException ignored) { - // Nothing we can do here. - } - } - /** Converts java.nio.file.Path objects to vfs.PathFragment. */ private final Function<Path, PathFragment> nioAbsolutePathToPathFragment = new Function<Path, PathFragment>() { - @Override - public PathFragment apply(Path input) { - Preconditions.checkArgument(input.startsWith(watchRootPath), "%s %s", input, - watchRootPath); - return new PathFragment(watchRootPath.relativize(input).toString()); - } - }; - - /** Returns the changed files caught by the watch service. */ - private Set<Path> collectChanges() throws BrokenDiffAwarenessException, IOException { - Set<Path> createdFilesAndDirectories = new HashSet<>(); - Set<Path> deletedOrModifiedFilesAndDirectories = new HashSet<>(); - Set<Path> deletedTrackedDirectories = new HashSet<>(); - - WatchKey watchKey; - while ((watchKey = watchService.poll()) != null) { - Path dir = watchKeyToDirBiMap.get(watchKey); - Preconditions.checkArgument(dir != null); - - // We replay all the events for this watched directory in chronological order and - // construct the diff of this directory since the last #collectChanges call. - for (WatchEvent<?> event : watchKey.pollEvents()) { - Kind<?> kind = event.kind(); - if (kind == StandardWatchEventKinds.OVERFLOW) { - // TODO(bazel-team): find out when an overflow might happen, and maybe handle it more - // gently. - throw new BrokenDiffAwarenessException("Overflow when watching local filesystem for " - + "changes"); + @Override + public PathFragment apply(Path input) { + Preconditions.checkArgument( + input.startsWith(watchRootPath), "%s %s", input, watchRootPath); + return new PathFragment(watchRootPath.relativize(input).toString()); } - if (event.context() == null) { - // The WatchService documentation mentions that WatchEvent#context may return null, but - // doesn't explain how/why it would do so. Looking at the implementation, it only - // happens on an overflow event. But we make no assumptions about that implementation - // detail here. - throw new BrokenDiffAwarenessException("Insufficient information from local file system " - + "watcher"); - } - // For the events we've registered, the context given is a relative path. - Path relativePath = (Path) event.context(); - Path path = dir.resolve(relativePath); - Preconditions.checkState(path.isAbsolute(), path); - if (kind == StandardWatchEventKinds.ENTRY_CREATE) { - createdFilesAndDirectories.add(path); - deletedOrModifiedFilesAndDirectories.remove(path); - } else if (kind == StandardWatchEventKinds.ENTRY_DELETE) { - createdFilesAndDirectories.remove(path); - deletedOrModifiedFilesAndDirectories.add(path); - WatchKey deletedDirectoryKey = watchKeyToDirBiMap.inverse().get(path); - if (deletedDirectoryKey != null) { - // If the deleted directory has children, then there will also be events for the - // WatchKey of the directory itself. WatchService#poll doesn't specify the order in - // which WatchKeys are returned, so the key for the directory itself may be processed - // *after* the current key (the parent of the deleted directory), and so we don't want - // to remove the deleted directory from our bimap just yet. - // - // For example, suppose we have the file '/root/a/foo.txt' and are watching the - // directories '/root' and '/root/a'. If the directory '/root/a' gets deleted then the - // following is a valid sequence of events by key. - // - // WatchKey '/root/' - // WatchEvent EVENT_MODIFY 'a' - // WatchEvent EVENT_DELETE 'a' - // WatchKey '/root/a' - // WatchEvent EVENT_DELETE 'foo.txt' - deletedTrackedDirectories.add(path); - // Since inotify uses inodes under the covers we cancel our registration on this key to - // avoid getting WatchEvents from a new directory that happens to have the same inode. - deletedDirectoryKey.cancel(); - } - } else if (kind == StandardWatchEventKinds.ENTRY_MODIFY) { - // If a file was created and then modified, then the net diff is that it was - // created. - if (!createdFilesAndDirectories.contains(path)) { - deletedOrModifiedFilesAndDirectories.add(path); - } - } - } - - if (!watchKey.reset()) { - // Watcher got deleted, directory no longer valid. - watchKeyToDirBiMap.remove(watchKey); - } - } - - for (Path path : deletedTrackedDirectories) { - WatchKey staleKey = watchKeyToDirBiMap.inverse().get(path); - watchKeyToDirBiMap.remove(staleKey); - } - if (watchKeyToDirBiMap.isEmpty()) { - // No more directories to watch, something happened the root directory being watched. - throw new IOException("Root directory " + watchRootPath + " became inaccessible."); - } - - Set<Path> changedPaths = new HashSet<>(); - for (Path path : createdFilesAndDirectories) { - if (Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { - // This is a new directory, so changes to it since its creation have not been watched. - // We manually traverse the directory tree to register all the new subdirectories and find - // all the new subdirectories and files. - changedPaths.addAll(registerSubDirectoriesAndReturnContents(path)); - } else { - changedPaths.add(path); - } - } - changedPaths.addAll(deletedOrModifiedFilesAndDirectories); - return changedPaths; - } - - /** - * Traverses directory tree to register subdirectories. Returns all paths traversed (as absolute - * paths). - */ - private Set<Path> registerSubDirectoriesAndReturnContents(Path rootDir) throws IOException { - Set<Path> visitedAbsolutePaths = new HashSet<>(); - // Note that this does not follow symlinks. - Files.walkFileTree(rootDir, new WatcherFileVisitor(visitedAbsolutePaths)); - return visitedAbsolutePaths; - } - - /** File visitor used by Files.walkFileTree() upon traversing subdirectories. */ - private class WatcherFileVisitor extends SimpleFileVisitor<Path> { - - private final Set<Path> visitedAbsolutePaths; - - private WatcherFileVisitor(Set<Path> visitedPaths) { - this.visitedAbsolutePaths = visitedPaths; - } - - @Override - public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) { - Preconditions.checkState(path.isAbsolute(), path); - visitedAbsolutePaths.add(path); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) - throws IOException { - // It's important that we register the directory before we visit its children. This way we - // are guaranteed to see new files/directories either on this #getDiff or the next one. - // Otherwise, e.g., an intra-build creation of a child directory will be forever missed if it - // happens before the directory is listed as part of the visitation. - WatchKey key = path.register(watchService, - StandardWatchEventKinds.ENTRY_CREATE, - StandardWatchEventKinds.ENTRY_MODIFY, - StandardWatchEventKinds.ENTRY_DELETE); - Preconditions.checkState(path.isAbsolute(), path); - visitedAbsolutePaths.add(path); - watchKeyToDirBiMap.put(key, path); - return FileVisitResult.CONTINUE; - } - } + }; } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/MacOSXFsEventsDiffAwareness.java b/src/main/java/com/google/devtools/build/lib/skyframe/MacOSXFsEventsDiffAwareness.java new file mode 100644 index 0000000000..cf0fc621b5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/MacOSXFsEventsDiffAwareness.java @@ -0,0 +1,108 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.UnixJniLoader; + +import java.io.File; +import java.nio.file.Path; + +/** + * A {@link DiffAwareness} that use fsevents to watch the filesystem to use in lieu of + * {@link LocalDiffAwareness}. + * + * <p> + * On OS X, the local diff awareness cannot work because WatchService is dummy and do polling, which + * is slow (https://bugs.openjdk.java.net/browse/JDK-7133447). + */ +public final class MacOSXFsEventsDiffAwareness extends LocalDiffAwareness { + + private boolean closed; + + // Keep a pointer to a native structure in the JNI code (the FsEvents callback needs that + // structure). + private long nativePointer; + + /** + * Watch changes on the file system under <code>watchRoot</code> with a granularity of + * <code>delay</code> seconds. + */ + MacOSXFsEventsDiffAwareness(String watchRoot, double latency) { + super(watchRoot); + create(new String[] {watchRootPath.toAbsolutePath().toString()}, latency); + + // Start a thread that just contains the OS X run loop. + new Thread( + new Runnable() { + @Override + public void run() { + MacOSXFsEventsDiffAwareness.this.run(); + } + }) + .start(); + } + + /** + * Watch changes on the file system under <code>watchRoot</code> with a granularity of 5ms. + */ + MacOSXFsEventsDiffAwareness(String watchRoot) { + this(watchRoot, 0.005); + } + + /** + * Helper function to start the watch of <code>paths</code>, called by the constructor. + */ + private native void create(String[] paths, double latency); + + /** + * Run the main loop + */ + private native void run(); + + /** + * Close this watch service, this service should not be used any longer after closing. + */ + public synchronized void close() { + Preconditions.checkState(!closed); + closed = true; + doClose(); + } + + /** + * JNI code stopping the main loop and shutting down listening to FSEvents. + */ + private synchronized native void doClose(); + + /** + * JNI code returning the list of absolute path modified since last call. + */ + private native String[] poll(); + + static { + UnixJniLoader.loadJni(); + } + + @Override + public synchronized View getCurrentView() throws BrokenDiffAwarenessException { + Preconditions.checkState(!closed); + ImmutableSet.Builder<Path> paths = ImmutableSet.builder(); + for (String path : poll()) { + paths.add(new File(path).toPath()); + } + return newView(paths.build()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WatchServiceDiffAwareness.java b/src/main/java/com/google/devtools/build/lib/skyframe/WatchServiceDiffAwareness.java new file mode 100644 index 0000000000..b6d274dc5a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/WatchServiceDiffAwareness.java @@ -0,0 +1,242 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.util.Preconditions; + +import java.io.IOException; +import java.nio.file.ClosedWatchServiceException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardWatchEventKinds; +import java.nio.file.WatchEvent; +import java.nio.file.WatchEvent.Kind; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashSet; +import java.util.Set; + +/** + * File system watcher for local filesystems. It's able to provide a list of changed files between + * two consecutive calls. Uses the standard Java WatchService, which uses 'inotify' on Linux. + */ +public final class WatchServiceDiffAwareness extends LocalDiffAwareness { + + /** + * Bijection from WatchKey to the (absolute) Path being watched. WatchKeys don't have this + * functionality built-in so we do it ourselves. + */ + private final HashBiMap<WatchKey, Path> watchKeyToDirBiMap = HashBiMap.create(); + + /** Every directory is registered under this watch service. */ + private WatchService watchService; + + WatchServiceDiffAwareness(String watchRoot, WatchService watchService) { + super(watchRoot); + this.watchService = watchService; + } + + @Override + public View getCurrentView() throws BrokenDiffAwarenessException { + Set<Path> modifiedAbsolutePaths; + if (isFirstCall()) { + try { + registerSubDirectoriesAndReturnContents(watchRootPath); + } catch (IOException e) { + close(); + throw new BrokenDiffAwarenessException( + "Error encountered with local file system watcher " + e); + } + modifiedAbsolutePaths = ImmutableSet.of(); + } else { + try { + modifiedAbsolutePaths = collectChanges(); + } catch (BrokenDiffAwarenessException e) { + close(); + throw e; + } catch (IOException e) { + close(); + throw new BrokenDiffAwarenessException( + "Error encountered with local file system watcher " + e); + } catch (ClosedWatchServiceException e) { + throw new BrokenDiffAwarenessException( + "Internal error with the local file system watcher " + e); + } + } + return newView(modifiedAbsolutePaths); + } + + @Override + public void close() { + try { + watchService.close(); + } catch (IOException ignored) { + // Nothing we can do here. + } + } + + /** Returns the changed files caught by the watch service. */ + private Set<Path> collectChanges() throws BrokenDiffAwarenessException, IOException { + Set<Path> createdFilesAndDirectories = new HashSet<>(); + Set<Path> deletedOrModifiedFilesAndDirectories = new HashSet<>(); + Set<Path> deletedTrackedDirectories = new HashSet<>(); + + WatchKey watchKey; + while ((watchKey = watchService.poll()) != null) { + Path dir = watchKeyToDirBiMap.get(watchKey); + Preconditions.checkArgument(dir != null); + + // We replay all the events for this watched directory in chronological order and + // construct the diff of this directory since the last #collectChanges call. + for (WatchEvent<?> event : watchKey.pollEvents()) { + Kind<?> kind = event.kind(); + if (kind == StandardWatchEventKinds.OVERFLOW) { + // TODO(bazel-team): find out when an overflow might happen, and maybe handle it more + // gently. + throw new BrokenDiffAwarenessException( + "Overflow when watching local filesystem for " + "changes"); + } + if (event.context() == null) { + // The WatchService documentation mentions that WatchEvent#context may return null, but + // doesn't explain how/why it would do so. Looking at the implementation, it only + // happens on an overflow event. But we make no assumptions about that implementation + // detail here. + throw new BrokenDiffAwarenessException( + "Insufficient information from local file system " + "watcher"); + } + // For the events we've registered, the context given is a relative path. + Path relativePath = (Path) event.context(); + Path path = dir.resolve(relativePath); + Preconditions.checkState(path.isAbsolute(), path); + if (kind == StandardWatchEventKinds.ENTRY_CREATE) { + createdFilesAndDirectories.add(path); + deletedOrModifiedFilesAndDirectories.remove(path); + } else if (kind == StandardWatchEventKinds.ENTRY_DELETE) { + createdFilesAndDirectories.remove(path); + deletedOrModifiedFilesAndDirectories.add(path); + WatchKey deletedDirectoryKey = watchKeyToDirBiMap.inverse().get(path); + if (deletedDirectoryKey != null) { + // If the deleted directory has children, then there will also be events for the + // WatchKey of the directory itself. WatchService#poll doesn't specify the order in + // which WatchKeys are returned, so the key for the directory itself may be processed + // *after* the current key (the parent of the deleted directory), and so we don't want + // to remove the deleted directory from our bimap just yet. + // + // For example, suppose we have the file '/root/a/foo.txt' and are watching the + // directories '/root' and '/root/a'. If the directory '/root/a' gets deleted then the + // following is a valid sequence of events by key. + // + // WatchKey '/root/' + // WatchEvent EVENT_MODIFY 'a' + // WatchEvent EVENT_DELETE 'a' + // WatchKey '/root/a' + // WatchEvent EVENT_DELETE 'foo.txt' + deletedTrackedDirectories.add(path); + // Since inotify uses inodes under the covers we cancel our registration on this key to + // avoid getting WatchEvents from a new directory that happens to have the same inode. + deletedDirectoryKey.cancel(); + } + } else if (kind == StandardWatchEventKinds.ENTRY_MODIFY) { + // If a file was created and then modified, then the net diff is that it was + // created. + if (!createdFilesAndDirectories.contains(path)) { + deletedOrModifiedFilesAndDirectories.add(path); + } + } + } + + if (!watchKey.reset()) { + // Watcher got deleted, directory no longer valid. + watchKeyToDirBiMap.remove(watchKey); + } + } + + for (Path path : deletedTrackedDirectories) { + WatchKey staleKey = watchKeyToDirBiMap.inverse().get(path); + watchKeyToDirBiMap.remove(staleKey); + } + if (watchKeyToDirBiMap.isEmpty()) { + // No more directories to watch, something happened the root directory being watched. + throw new IOException("Root directory " + watchRootPath + " became inaccessible."); + } + + Set<Path> changedPaths = new HashSet<>(); + for (Path path : createdFilesAndDirectories) { + if (Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { + // This is a new directory, so changes to it since its creation have not been watched. + // We manually traverse the directory tree to register all the new subdirectories and find + // all the new subdirectories and files. + changedPaths.addAll(registerSubDirectoriesAndReturnContents(path)); + } else { + changedPaths.add(path); + } + } + changedPaths.addAll(deletedOrModifiedFilesAndDirectories); + return changedPaths; + } + + /** + * Traverses directory tree to register subdirectories. Returns all paths traversed (as absolute + * paths). + */ + private Set<Path> registerSubDirectoriesAndReturnContents(Path rootDir) throws IOException { + Set<Path> visitedAbsolutePaths = new HashSet<>(); + // Note that this does not follow symlinks. + Files.walkFileTree(rootDir, new WatcherFileVisitor(visitedAbsolutePaths)); + return visitedAbsolutePaths; + } + + /** File visitor used by Files.walkFileTree() upon traversing subdirectories. */ + private class WatcherFileVisitor extends SimpleFileVisitor<Path> { + + private final Set<Path> visitedAbsolutePaths; + + private WatcherFileVisitor(Set<Path> visitedPaths) { + this.visitedAbsolutePaths = visitedPaths; + } + + @Override + public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) { + Preconditions.checkState(path.isAbsolute(), path); + visitedAbsolutePaths.add(path); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) + throws IOException { + // It's important that we register the directory before we visit its children. This way we + // are guaranteed to see new files/directories either on this #getDiff or the next one. + // Otherwise, e.g., an intra-build creation of a child directory will be forever missed if it + // happens before the directory is listed as part of the visitation. + WatchKey key = + path.register( + watchService, + StandardWatchEventKinds.ENTRY_CREATE, + StandardWatchEventKinds.ENTRY_MODIFY, + StandardWatchEventKinds.ENTRY_DELETE); + Preconditions.checkState(path.isAbsolute(), path); + visitedAbsolutePaths.add(path); + watchKeyToDirBiMap.put(key, path); + return FileVisitResult.CONTINUE; + } + } +} diff --git a/src/main/native/BUILD b/src/main/native/BUILD index d1b74356eb..65b6bb5672 100644 --- a/src/main/native/BUILD +++ b/src/main/native/BUILD @@ -20,8 +20,14 @@ genrule( filegroup( name = "jni_os", srcs = select({ - "//src:darwin": ["unix_jni_darwin.cc"], - "//src:darwin_x86_64": ["unix_jni_darwin.cc"], + "//src:darwin": [ + "unix_jni_darwin.cc", + "fsevents.cc", + ], + "//src:darwin_x86_64": [ + "unix_jni_darwin.cc", + "fsevents.cc", + ], "//src:freebsd": ["unix_jni_freebsd.cc"], "//conditions:default": ["unix_jni_linux.cc"], }), @@ -44,6 +50,11 @@ cc_binary( "-DBLAZE_JAVA_CPU=\"k8\"", ], includes = ["."], # For jni headers. + linkopts = select({ + "//src:darwin": ["-framework CoreServices"], + "//src:darwin_x86_64": ["-framework CoreServices"], + "//conditions:default": [], + }), linkshared = 1, visibility = ["//src:__subpackages__"], deps = [ diff --git a/src/main/native/fsevents.cc b/src/main/native/fsevents.cc new file mode 100644 index 0000000000..1c1cac4ad5 --- /dev/null +++ b/src/main/native/fsevents.cc @@ -0,0 +1,150 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include <CoreServices/CoreServices.h> +#include <jni.h> +#include <pthread.h> +#include <stdlib.h> +#include <list> +#include <string> + +// A structure to pass around the FSEvents info and the list of paths. +struct JNIEventsDiffAwareness { + // FSEvents run loop (thread) + CFRunLoopRef runLoop; + // FSEvents stream reference (reference to the listened stream) + FSEventStreamRef stream; + // List of paths that have been changed since last polling + std::list<std::string> paths; + // Mutex to protect concurrent access of paths. + // FsEventsDiffAwarenessCallback fill that list which is emptied + // by the MacOSXEventsDiffAwareness#poll() method. + // The former is called inside the FsEvents run loop and the latter + // from Java threads. + pthread_mutex_t mutex; +}; + +// Callback called when an event is reported by the FSEvents API +void FsEventsDiffAwarenessCallback(ConstFSEventStreamRef streamRef, + void *clientCallBackInfo, size_t numEvents, + void *eventPaths, + const FSEventStreamEventFlags eventFlags[], + const FSEventStreamEventId eventIds[]) { + /* We are just returning the list of modified path but we could return a bit + * more information, + * see + * https://developer.apple.com/library/mac/documentation/Darwin/Reference/FSEvents_Ref/#//apple_ref/doc/c_ref/FSEventStreamCallback + * If we ever do more, we should be careful because creation and deletion + * event get coaslesced + * into one single entry. + */ + char **paths = static_cast<char **>(eventPaths); + + JNIEventsDiffAwareness *info = + static_cast<JNIEventsDiffAwareness *>(clientCallBackInfo); + pthread_mutex_lock(&(info->mutex)); + for (int i = 0; i < numEvents; i++) { + info->paths.push_back(std::string(paths[i])); + } + pthread_mutex_unlock(&(info->mutex)); +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_devtools_build_lib_skyframe_MacOSXFsEventsDiffAwareness_create( + JNIEnv *env, jobject fsEventsDiffAwareness, jobjectArray paths, + jdouble latency) { + // Create a FSEventStreamContext to pass around (env, fsEventsDiffAwareness) + JNIEventsDiffAwareness *info = new JNIEventsDiffAwareness; + + FSEventStreamContext context; + context.version = 0; + context.info = static_cast<void *>(info); + context.retain = NULL; + context.release = NULL; + context.copyDescription = NULL; + + // Create an CFArrayRef of CFStringRef from the Java array of String + jsize length = env->GetArrayLength(paths); + CFStringRef *pathsArray = new CFStringRef[length]; + for (int i = 0; i < length; i++) { + jstring path = (jstring)env->GetObjectArrayElement(paths, i); + const char *pathCStr = env->GetStringUTFChars(path, NULL); + pathsArray[i] = + CFStringCreateWithCString(NULL, pathCStr, kCFStringEncodingUTF8); + env->ReleaseStringUTFChars(path, pathCStr); + } + CFArrayRef pathsToWatch = + CFArrayCreate(NULL, (const void **)pathsArray, 1, NULL); + delete pathsArray; + info->stream = FSEventStreamCreate( + NULL, &FsEventsDiffAwarenessCallback, &context, pathsToWatch, + kFSEventStreamEventIdSinceNow, static_cast<CFAbsoluteTime>(latency), + kFSEventStreamCreateFlagFileEvents); + + // Save the info pointer to FSEventsDiffAwareness#nativePointer + jbyteArray array = env->NewByteArray(sizeof(info)); + env->SetByteArrayRegion(array, 0, sizeof(info), + reinterpret_cast<const jbyte *>(&info)); + jclass clazz = env->GetObjectClass(fsEventsDiffAwareness); + jfieldID fid = env->GetFieldID(clazz, "nativePointer", "J"); + env->SetLongField(fsEventsDiffAwareness, fid, reinterpret_cast<jlong>(info)); +} + +JNIEventsDiffAwareness *GetInfo(JNIEnv *env, jobject fsEventsDiffAwareness) { + jclass clazz = env->GetObjectClass(fsEventsDiffAwareness); + jfieldID fid = env->GetFieldID(clazz, "nativePointer", "J"); + jlong field = env->GetLongField(fsEventsDiffAwareness, fid); + return reinterpret_cast<JNIEventsDiffAwareness *>(field); +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_devtools_build_lib_skyframe_MacOSXFsEventsDiffAwareness_run( + JNIEnv *env, jobject fsEventsDiffAwareness) { + JNIEventsDiffAwareness *info = GetInfo(env, fsEventsDiffAwareness); + info->runLoop = CFRunLoopGetCurrent(); + FSEventStreamScheduleWithRunLoop(info->stream, info->runLoop, + kCFRunLoopDefaultMode); + FSEventStreamStart(info->stream); + CFRunLoopRun(); +} + +extern "C" JNIEXPORT jobjectArray JNICALL +Java_com_google_devtools_build_lib_skyframe_MacOSXFsEventsDiffAwareness_poll( + JNIEnv *env, jobject fsEventsDiffAwareness) { + JNIEventsDiffAwareness *info = GetInfo(env, fsEventsDiffAwareness); + pthread_mutex_lock(&(info->mutex)); + + jclass classString = env->FindClass("java/lang/String"); + jobjectArray result = + env->NewObjectArray(info->paths.size(), classString, NULL); + int i = 0; + for (auto it = info->paths.begin(); it != info->paths.end(); it++, i++) { + env->SetObjectArrayElement(result, i, env->NewStringUTF(it->c_str())); + } + info->paths.clear(); + pthread_mutex_unlock(&(info->mutex)); + return result; +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_devtools_build_lib_skyframe_MacOSXFsEventsDiffAwareness_doClose( + JNIEnv *env, jobject fsEventsDiffAwareness) { + JNIEventsDiffAwareness *info = GetInfo(env, fsEventsDiffAwareness); + CFRunLoopStop(info->runLoop); + FSEventStreamStop(info->stream); + FSEventStreamUnscheduleFromRunLoop(info->stream, info->runLoop, + kCFRunLoopDefaultMode); + FSEventStreamInvalidate(info->stream); + FSEventStreamRelease(info->stream); + delete info; +} |