From d08b27fa9701fecfdb69e1b0d1ac2459efc2129b Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 25 Feb 2015 16:45:20 +0100 Subject: Update from Google. -- MOE_MIGRATED_REVID=85702957 --- .../lib/rules/fileset/FilesetActionContext.java | 34 ++++ .../rules/fileset/FilesetActionContextImpl.java | 101 ++++++++++ .../build/lib/rules/fileset/FilesetLinks.java | 218 +++++++++++++++++++++ .../build/lib/rules/fileset/FilesetProvider.java | 27 +++ .../build/lib/rules/fileset/SymlinkTraversal.java | 54 +++++ 5 files changed, 434 insertions(+) create mode 100644 src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContext.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContextImpl.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetLinks.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetProvider.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/fileset/SymlinkTraversal.java (limited to 'src/main/java/com/google/devtools/build/lib/rules/fileset') diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContext.java new file mode 100644 index 0000000000..056b61e4d4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContext.java @@ -0,0 +1,34 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.fileset; + +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +import java.util.concurrent.ThreadPoolExecutor; + +/** + * Action context for fileset collection actions. + */ +public interface FilesetActionContext extends ActionContext { + + /** + * Returns a thread pool for fileset symlink tree creation. + */ + ThreadPoolExecutor getFilesetPool(); + + /** + * Returns the name of the workspace the build is run in. + */ + String getWorkspaceName(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContextImpl.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContextImpl.java new file mode 100644 index 0000000000..9c03129758 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContextImpl.java @@ -0,0 +1,101 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.fileset; + +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.devtools.build.lib.actions.ActionContextProvider; +import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BlazeExecutor; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.events.Reporter; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Context for Fileset manifest actions. It currently only provides a ThreadPoolExecutor. + * + *

Fileset is a legacy, google-internal mechanism to make parts of the source tree appear as a + * tree in the output directory. + */ +@ExecutionStrategy(contextType = FilesetActionContext.class) +public final class FilesetActionContextImpl implements FilesetActionContext { + // TODO(bazel-team): it would be nice if this weren't shipped in Bazel at all. + + /** + * Factory class. + */ + public static class Provider implements ActionContextProvider { + private FilesetActionContextImpl impl; + private final Reporter reporter; + private final ThreadPoolExecutor filesetPool; + + public Provider(Reporter reporter, String workspaceName) { + this.reporter = reporter; + this.filesetPool = newFilesetPool(100); + this.impl = new FilesetActionContextImpl(filesetPool, workspaceName); + } + + private static ThreadPoolExecutor newFilesetPool(int threads) { + ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 3L, TimeUnit.SECONDS, + new LinkedBlockingQueue()); + // Do not consume threads when not in use. + pool.allowCoreThreadTimeOut(true); + pool.setThreadFactory(new ThreadFactoryBuilder().setNameFormat("Fileset worker %d").build()); + return pool; + } + + @Override + public Iterable getActionContexts() { + return ImmutableList.of(impl); + } + + @Override + public void executorCreated(Iterable usedStrategies) {} + + @Override + public void executionPhaseStarting( + ActionInputFileCache actionInputFileCache, + ActionGraph actionGraph, + Iterable topLevelArtifacts) {} + + @Override + public void executionPhaseEnding() { + BlazeExecutor.shutdownHelperPool(reporter, filesetPool, "Fileset"); + } + } + + private final ThreadPoolExecutor filesetPool; + private final String workspaceName; + + private FilesetActionContextImpl(ThreadPoolExecutor filesetPool, String workspaceName) { + this.filesetPool = filesetPool; + this.workspaceName = workspaceName; + } + + @Override + public ThreadPoolExecutor getFilesetPool() { + return filesetPool; + } + + @Override + public String getWorkspaceName() { + return workspaceName; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetLinks.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetLinks.java new file mode 100644 index 0000000000..d523edc356 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetLinks.java @@ -0,0 +1,218 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.fileset; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.syntax.FilesetEntry; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * FilesetLinks manages the set of links added to a Fileset. If two links conflict, the first wins. + * + *

FilesetLinks is FileSystem-aware. For example, if you first create a link + * (a/b/c, foo), a subsequent call to link (a/b, bar) is a no-op. + * This is because the first link requires us to create a directory "a/b", + * so "a/b" cannot also link to "bar". + * + *

TODO(bazel-team): Consider warning if we have such a conflict; we don't do that currently. + */ +public interface FilesetLinks { + + /** + * Get late directory information for a source. + * + * @param src The source to search for. + * @return The late directory info, or null if none was found. + */ + public LateDirectoryInfo getLateDirectoryInfo(PathFragment src); + + public boolean putLateDirectoryInfo(PathFragment src, LateDirectoryInfo lateDir); + + /** + * Add specified file as a symlink. + * + * The behavior when the target file is a symlink depends on the + * symlinkBehavior parameter (see comments for FilesetEntry.SymlinkBehavior). + * + * @param src The root-relative symlink path. + * @param target The symlink target. + */ + public void addFile(PathFragment src, Path target, String metadata, + FilesetEntry.SymlinkBehavior symlinkBehavior) + throws IOException; + + /** + * Add all late directories as symlinks. This function should be called only + * after all recursions have completed, but before getData or getSymlinks are + * called. + */ + public void addLateDirectories() throws IOException; + + /** + * Adds the given symlink to the tree. + * + * @param fromFrag The root-relative symlink path. + * @param toFrag The symlink target. + * @return true iff the symlink was added. + */ + public boolean addLink(PathFragment fromFrag, PathFragment toFrag, String dataVal); + + /** + * @return The unmodifiable map of symlinks. + */ + public Map getSymlinks(); + + /** + * @return The unmodifiable map of metadata. + */ + public Map getData(); + + /** + * A data structure for containing all the information about a directory that + * is late-added. This means the directory is skipped unless we need to + * recurse into it later. If the directory is never recursed into, we will + * create a symlink directly to it. + */ + public static final class LateDirectoryInfo { + // The constructors are private. Use the factory functions below to create + // instances of this class. + + /** Construct a stub LateDirectoryInfo object. */ + private LateDirectoryInfo() { + this.added = new AtomicBoolean(true); + + // Shut up the compiler. + this.target = null; + this.src = null; + this.pkgMode = SubpackageMode.IGNORE; + this.metadata = null; + this.symlinkBehavior = null; + } + + /** Construct a normal LateDirectoryInfo object. */ + private LateDirectoryInfo(Path target, PathFragment src, SubpackageMode pkgMode, + String metadata, FilesetEntry.SymlinkBehavior symlinkBehavior) { + this.target = target; + this.src = src; + this.pkgMode = pkgMode; + this.metadata = metadata; + this.symlinkBehavior = symlinkBehavior; + this.added = new AtomicBoolean(false); + } + + /** @return The target path for the symlink. The target is the referent. */ + public Path getTarget() { + return target; + } + + /** + * @return The source path for the symlink. The source is the place the + * symlink will be written. */ + public PathFragment getSrc() { + return src; + } + + /** + * @return Whether we should show a warning if we cross a package boundary + * when recursing into this directory. + */ + public SubpackageMode getPkgMode() { + return pkgMode; + } + + /** + * @return The metadata we will write into the manifest if we symlink to + * this directory. + */ + public String getMetadata() { + return metadata; + } + + /** + * @return How to perform the symlinking if the source happens to be a + * symlink itself. + */ + public FilesetEntry.SymlinkBehavior getTargetSymlinkBehavior() { + return Preconditions.checkNotNull(symlinkBehavior, + "should not call this method on stub instances"); + } + + /** + * Atomically checks if the late directory has been added to the manifest + * and marks it as added. If this function returns true, it is the + * responsibility of the caller to recurse into the late directory. + * Otherwise, some other caller has already, or is in the process of + * recursing into it. + * @return Whether the caller should recurse into the late directory. + */ + public boolean shouldAdd() { + return !added.getAndSet(true); + } + + /** + * Create a stub LateDirectoryInfo that is already marked as added. + * @return The new LateDirectoryInfo object. + */ + public static LateDirectoryInfo createStub() { + return new LateDirectoryInfo(); + } + + /** + * Create a LateDirectoryInfo object with the specified attributes. + * @param target The directory to which the symlinks will refer. + * @param src The location at which to create the symlink. + * @param pkgMode How to handle recursion into another package. + * @param metadata The metadata for the directory to write into the + * manifest if we symlink it directly. + * @return The new LateDirectoryInfo object. + */ + public static LateDirectoryInfo create(Path target, PathFragment src, SubpackageMode pkgMode, + String metadata, FilesetEntry.SymlinkBehavior symlinkBehavior) { + return new LateDirectoryInfo(target, src, pkgMode, metadata, symlinkBehavior); + } + + /** + * The target directory to which the symlink will point. + * Note this is a real path on the filesystem and can't be compared to src + * or any source (key) in the links map. + */ + private final Path target; + + /** The referent of the symlink. */ + private final PathFragment src; + + /** Whether to show cross package boundary warnings / errors. */ + private final SubpackageMode pkgMode; + + /** The metadata to write into the manifest file. */ + private final String metadata; + + /** How to perform the symlinking if the source happens to be a symlink itself. */ + private final FilesetEntry.SymlinkBehavior symlinkBehavior; + + /** Whether the directory has already been recursed into. */ + private final AtomicBoolean added; + } + + /** How to handle filesets that cross subpackages. */ + public static enum SubpackageMode { + ERROR, WARNING, IGNORE; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetProvider.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetProvider.java new file mode 100644 index 0000000000..6b70aab267 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetProvider.java @@ -0,0 +1,27 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.fileset; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Information needed by a Fileset to do the right thing when it depends on another Fileset. + */ +public interface FilesetProvider extends TransitiveInfoProvider { + Artifact getFilesetInputManifest(); + PathFragment getFilesetLinkDir(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/SymlinkTraversal.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/SymlinkTraversal.java new file mode 100644 index 0000000000..db13bdb3c4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/SymlinkTraversal.java @@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.fileset; + +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; + +import java.io.IOException; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * An interface which contains a method to compute a symlink mapping. + */ +public interface SymlinkTraversal { + + /** + * Adds symlinks to the given FilesetLinks. + * + * @throws IOException if a filesystem operation fails. + * @throws InterruptedException if the traversal is interrupted. + */ + void addSymlinks(EventHandler eventHandler, FilesetLinks links, ThreadPoolExecutor filesetPool) + throws IOException, InterruptedException; + + /** + * Add the traversal's fingerprint to the given Fingerprint. + * @param fp the Fingerprint to combine. + */ + void fingerprint(Fingerprint fp); + + /** + * @return true iff this traversal must be executed unconditionally. + */ + boolean executeUnconditionally(); + + /** + * Returns true if it's ever possible that {@link #executeUnconditionally} + * could evaluate to true during the lifetime of this instance, false + * otherwise. + */ + boolean isVolatile(); +} -- cgit v1.2.3