// Copyright 2014 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.vfs; import static java.nio.charset.StandardCharsets.ISO_8859_1; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.io.ByteSink; import com.google.common.io.ByteSource; import com.google.common.io.ByteStreams; import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; /** * Helper functions that implement often-used complex operations on file * systems. */ @ConditionallyThreadSafe // ThreadSafe except for deleteTree. public class FileSystemUtils { private FileSystemUtils() {} /**************************************************************************** * Path and PathFragment functions. */ /** * Throws exceptions if {@code baseName} is not a valid base name. A valid * base name: *
e.g. "foo/bar.x" -> "foo/bar" * *
Note that if the filename is composed entirely of ".", this method will return the string * with one fewer ".", which may have surprising effects. */ @ThreadSafe public static String removeExtension(String filename) { int lastDotIndex = filename.lastIndexOf('.'); if (lastDotIndex == -1) { return filename; } int lastSlashIndex = filename.lastIndexOf('/'); if (lastSlashIndex > lastDotIndex) { return filename; } return filename.substring(0, lastDotIndex); } /** * Removes the shortest suffix beginning with '.' from the basename of the * PathFragment. If the basename contains no '.', the filename is returned * unchanged. * *
e.g. "foo/bar.x" -> "foo/bar" * *
Note that if the base filename is composed entirely of ".", this method will return the * filename with one fewer "." in the base filename, which may have surprising effects. */ @ThreadSafe public static PathFragment removeExtension(PathFragment path) { return path.replaceName(removeExtension(path.getBaseName())); } /** * Removes the shortest suffix beginning with '.' from the basename of the * Path. If the basename contains no '.', the filename is returned * unchanged. * *
e.g. "foo/bar.x" -> "foo/bar" * *
Note that if the base filename is composed entirely of ".", this method will return the * filename with one fewer "." in the base filename, which may have surprising effects. */ @ThreadSafe public static Path removeExtension(Path path) { return path.getFileSystem().getPath(removeExtension(path.asFragment())); } /** * Returns a new {@code PathFragment} formed by replacing the extension of the * last path segment of {@code path} with {@code newExtension}. Null is * returned iff {@code path} has zero segments. */ public static PathFragment replaceExtension(PathFragment path, String newExtension) { return path.replaceName(removeExtension(path.getBaseName()) + newExtension); } /** * Returns a new {@code PathFragment} formed by replacing the extension of the * last path segment of {@code path} with {@code newExtension}. Null is * returned iff {@code path} has zero segments or it doesn't end with {@code oldExtension}. */ public static PathFragment replaceExtension(PathFragment path, String newExtension, String oldExtension) { String base = path.getBaseName(); if (!base.endsWith(oldExtension)) { return null; } String newBase = base.substring(0, base.length() - oldExtension.length()) + newExtension; return path.replaceName(newBase); } /** * Returns a new {@code Path} formed by replacing the extension of the * last path segment of {@code path} with {@code newExtension}. Null is * returned iff {@code path} has zero segments. */ public static Path replaceExtension(Path path, String newExtension) { PathFragment fragment = replaceExtension(path.asFragment(), newExtension); return fragment == null ? null : path.getFileSystem().getPath(fragment); } /** * Returns a new {@code PathFragment} formed by adding the extension to the last path segment of * {@code path}. Null is returned if {@code path} has zero segments. */ public static PathFragment appendExtension(PathFragment path, String newExtension) { return path.replaceName(path.getBaseName() + newExtension); } /** * Returns a new {@code PathFragment} formed by appending the given string to the last path * segment of {@code path} without removing the extension. Returns null if {@code path} * has zero segments. */ public static PathFragment appendWithoutExtension(PathFragment path, String toAppend) { return path.replaceName(appendWithoutExtension(path.getBaseName(), toAppend)); } /** * Given a string that represents a file with an extension separated by a '.' and a string * to append, return a string in which {@code toAppend} has been appended to {@code name} * before the last '.' character. If {@code name} does not include a '.', appends {@code * toAppend} at the end. * *
For example, * ("libfoo.jar", "-src") ==> "libfoo-src.jar" * ("libfoo", "-src") ==> "libfoo-src" */ private static String appendWithoutExtension(String name, String toAppend) { int dotIndex = name.lastIndexOf('.'); if (dotIndex > 0) { String baseName = name.substring(0, dotIndex); String extension = name.substring(dotIndex); return baseName + toAppend + extension; } else { return name + toAppend; } } /**************************************************************************** * FileSystem property functions. */ /** * Return the current working directory as expressed by the System property * 'user.dir'. */ public static Path getWorkingDirectory(FileSystem fs) { return fs.getPath(getWorkingDirectory()); } /** * Returns the current working directory as expressed by the System property * 'user.dir'. This version does not require a {@link FileSystem}. */ public static PathFragment getWorkingDirectory() { return PathFragment.create(System.getProperty("user.dir", "/")); } /**************************************************************************** * Path FileSystem mutating operations. */ /** * "Touches" the file or directory specified by the path, following symbolic * links. If it does not exist, it is created as an empty file; otherwise, the * time of last access is updated to the current time. * * @throws IOException if there was an error while touching the file */ @ThreadSafe public static void touchFile(Path path) throws IOException { if (path.exists()) { // -1L means "use the current time", and is ultimately implemented by // utime(path, null), thereby using the kernel's clock, not the JVM's. // (A previous implementation based on the JVM clock was found to be // skewy.) path.setLastModifiedTime(-1L); } else { createEmptyFile(path); } } /** * Creates an empty regular file with the name of the current path, following * symbolic links. * * @throws IOException if the file could not be created for any reason * (including that there was already a file at that location) */ public static void createEmptyFile(Path path) throws IOException { path.getOutputStream().close(); } /** * Creates or updates a symbolic link from 'link' to 'target'. Replaces * existing symbolic links with target, and skips the link creation if it is * already present. Will also create any missing ancestor directories of the * link. This method is non-atomic * *
Note: this method will throw an IOException if there is an unequal * non-symlink at link. * * @throws IOException if the creation of the symbolic link was unsuccessful * for any reason. */ @ThreadSafe // but not atomic public static void ensureSymbolicLink(Path link, Path target) throws IOException { ensureSymbolicLink(link, target.asFragment()); } /** * Creates or updates a symbolic link from 'link' to 'target'. Replaces * existing symbolic links with target, and skips the link creation if it is * already present. Will also create any missing ancestor directories of the * link. This method is non-atomic * *
Note: this method will throw an IOException if there is an unequal * non-symlink at link. * * @throws IOException if the creation of the symbolic link was unsuccessful * for any reason. */ @ThreadSafe // but not atomic public static void ensureSymbolicLink(Path link, String target) throws IOException { ensureSymbolicLink(link, PathFragment.create(target)); } /** * Creates or updates a symbolic link from 'link' to 'target'. Replaces * existing symbolic links with target, and skips the link creation if it is * already present. Will also create any missing ancestor directories of the * link. This method is non-atomic * *
Note: this method will throw an IOException if there is an unequal * non-symlink at link. * * @throws IOException if the creation of the symbolic link was unsuccessful * for any reason. */ @ThreadSafe // but not atomic public static void ensureSymbolicLink(Path link, PathFragment target) throws IOException { // TODO(bazel-team): (2009) consider adding the logic for recovering from the case when // we have already created a parent directory symlink earlier. try { if (link.readSymbolicLink().equals(target)) { return; // Do nothing if the link is already there. } } catch (IOException e) { // link missing or broken /* fallthru and do the work below */ } if (link.isSymbolicLink()) { link.delete(); // Remove the symlink since it is pointing somewhere else. } else { createDirectoryAndParents(link.getParentDirectory()); } try { link.createSymbolicLink(target); } catch (IOException e) { // Only pass on exceptions caused by a true link creation failure. if (!link.isSymbolicLink() || !link.resolveSymbolicLinks().equals(link.getRelative(target))) { throw e; } } } public static ByteSource asByteSource(final Path path) { return new ByteSource() { @Override public InputStream openStream() throws IOException { return path.getInputStream(); } }; } public static ByteSink asByteSink(final Path path, final boolean append) { return new ByteSink() { @Override public OutputStream openStream() throws IOException { return path.getOutputStream(append); } }; } public static ByteSink asByteSink(final Path path) { return asByteSink(path, false); } /** * Copies the file from location "from" to location "to", while overwriting a * potentially existing "to". File's last modified time, executable and * writable bits are also preserved. * *
If no error occurs, the method returns normally. If a parent directory does * not exist, a FileNotFoundException is thrown. An IOException is thrown when * other erroneous situations occur. (e.g. read errors) */ @ThreadSafe // but not atomic public static void copyFile(Path from, Path to) throws IOException { try { to.delete(); } catch (IOException e) { throw new IOException("error copying file: " + "couldn't delete destination: " + e.getMessage()); } asByteSource(from).copyTo(asByteSink(to)); to.setLastModifiedTime(from.getLastModifiedTime()); // Preserve mtime. if (!from.isWritable()) { to.setWritable(false); // Make file read-only if original was read-only. } to.setExecutable(from.isExecutable()); // Copy executable bit. } /** * Moves the file from location "from" to location "to", while overwriting a * potentially existing "to". File's last modified time, executable and * writable bits are also preserved. * *
If no error occurs, the method returns normally. If a parent directory does
* not exist, a FileNotFoundException is thrown. An IOException is thrown when
* other erroneous situations occur. (e.g. read errors)
*/
@ThreadSafe // but not atomic
public static void moveFile(Path from, Path to) throws IOException {
long mtime = from.getLastModifiedTime();
boolean writable = from.isWritable();
boolean executable = from.isExecutable();
// We don't try-catch here for better performance.
to.delete();
try {
from.renameTo(to);
} catch (IOException e) {
asByteSource(from).copyTo(asByteSink(to));
if (!from.delete()) {
if (!to.delete()) {
throw new IOException("Unable to delete " + to);
}
throw new IOException("Unable to delete " + from);
}
}
to.setLastModifiedTime(mtime); // Preserve mtime.
if (!writable) {
to.setWritable(false); // Make file read-only if original was read-only.
}
to.setExecutable(executable); // Copy executable bit.
}
/**
* Copies a tool binary from one path to another, returning the target path.
* The directory of the target path must already exist. The target copy's time
* is set to match, as well as its read-only and executable flags. The
* operation is skipped if the target file has the same time and size as the
* source.
*/
public static Path copyTool(Path source, Path target) throws IOException {
FileStatus sourceStat = null;
FileStatus targetStat = target.statNullable();
if (targetStat != null) {
// stat the source file only if we'll need the stat.
sourceStat = source.stat(Symlinks.FOLLOW);
}
if (targetStat == null ||
targetStat.getLastModifiedTime() != sourceStat.getLastModifiedTime() ||
targetStat.getSize() != sourceStat.getSize()) {
copyFile(source, target);
target.setWritable(source.isWritable());
target.setExecutable(source.isExecutable());
target.setLastModifiedTime(source.getLastModifiedTime());
}
return target;
}
/****************************************************************************
* Directory tree operations.
*/
/**
* Returns a new collection containing all of the paths below a given root
* path, for which the given predicate is true. Symbolic links are not
* followed, and may appear in the result.
*
* @throws IOException If the root does not denote a directory
*/
@ThreadSafe
public static Collection The source and the destination must be non-overlapping, otherwise an
* IllegalArgumentException will be thrown. This method cannot be used to copy a dir tree to a sub
* tree of itself.
*
* If no error occurs, the method returns normally. If the given 'from' does not exist, a
* FileNotFoundException is thrown. An IOException is thrown when other erroneous situations
* occur. (e.g. read errors)
*/
@ThreadSafe
public static void copyTreesBelow(Path from, Path to, Symlinks followSymlinks)
throws IOException {
if (to.startsWith(from)) {
throw new IllegalArgumentException(to + " is a subdirectory of " + from);
}
Collection The source and the destination must be non-overlapping, otherwise an
* IllegalArgumentException will be thrown. This method cannot be used to copy
* a dir tree to a sub tree of itself.
*
* If no error occurs, the method returns normally. If the given 'from' does
* not exist, a FileNotFoundException is thrown. An IOException is thrown when
* other erroneous situations occur. (e.g. read errors)
*/
@ThreadSafe
public static void moveTreesBelow(Path from , Path to) throws IOException {
if (to.startsWith(from)) {
throw new IllegalArgumentException(to + " is a subdirectory of " + from);
}
Collection Deprecated. Prefer to call {@link Path#createDirectoryAndParents()} directly.
*/
@Deprecated
@ThreadSafe
public static void createDirectoryAndParents(Path dir) throws IOException {
dir.createDirectoryAndParents();
}
/**
* Attempts to remove a relative chain of directories under a given base.
* Returns {@code true} if the removal was successful, and returns {@code
* false} if the removal fails because a directory was not empty. An
* {@link IOException} is thrown for any other errors.
*/
@ThreadSafe
public static boolean removeDirectoryAndParents(Path base, PathFragment toRemove) {
if (toRemove.isAbsolute()) {
return false;
}
try {
for (; toRemove.segmentCount() > 0; toRemove = toRemove.getParentDirectory()) {
Path p = base.getRelative(toRemove);
if (p.exists()) {
p.delete();
}
}
} catch (IOException e) {
return false;
}
return true;
}
/****************************************************************************
* Whole-file I/O utilities for characters and bytes. These convenience
* methods are not efficient and should not be used for large amounts of data!
*/
/**
* Decodes the given byte array assumed to be encoded with ISO-8859-1 encoding (isolatin1).
*/
public static char[] convertFromLatin1(byte[] content) {
char[] latin1 = new char[content.length];
for (int i = 0; i < latin1.length; i++) { // yeah, latin1 is this easy! :-)
latin1[i] = (char) (0xff & content[i]);
}
return latin1;
}
/**
* Writes lines to file using ISO-8859-1 encoding (isolatin1).
*/
@ThreadSafe // but not atomic
public static void writeIsoLatin1(Path file, String... lines) throws IOException {
writeLinesAs(file, ISO_8859_1, lines);
}
/**
* Append lines to file using ISO-8859-1 encoding (isolatin1).
*/
@ThreadSafe // but not atomic
public static void appendIsoLatin1(Path file, String... lines) throws IOException {
appendLinesAs(file, ISO_8859_1, lines);
}
/**
* Writes the specified String as ISO-8859-1 (latin1) encoded bytes to the
* file. Follows symbolic links.
*
* @throws IOException if there was an error
*/
public static void writeContentAsLatin1(Path outputFile, String content) throws IOException {
writeContent(outputFile, ISO_8859_1, content);
}
/**
* Writes the specified String using the specified encoding to the file.
* Follows symbolic links.
*
* @throws IOException if there was an error
*/
public static void writeContent(Path outputFile, Charset charset, String content)
throws IOException {
asByteSink(outputFile).asCharSink(charset).write(content);
}
/**
* Writes lines to file using the given encoding, ending every line with a
* line break '\n' character.
*/
@ThreadSafe // but not atomic
public static void writeLinesAs(Path file, Charset charset, String... lines)
throws IOException {
writeLinesAs(file, charset, Arrays.asList(lines));
}
/**
* Appends lines to file using the given encoding, ending every line with a
* line break '\n' character.
*/
@ThreadSafe // but not atomic
public static void appendLinesAs(Path file, Charset charset, String... lines)
throws IOException {
appendLinesAs(file, charset, Arrays.asList(lines));
}
/**
* Writes lines to file using the given encoding, ending every line with a
* line break '\n' character.
*/
@ThreadSafe // but not atomic
public static void writeLinesAs(Path file, Charset charset, Iterable If the output file already exists but is unreadable, this tries to overwrite it with the new
* contents. In other words: unreadable or missing files are considered to be non-matching.
*
* @throws IOException if there was an error
*/
public static void maybeUpdateContent(Path outputFile, byte[] newContent) throws IOException {
byte[] currentContent;
try {
currentContent = readContent(outputFile);
} catch (IOException e) {
// Ignore error per the rationale given in the docstring. Keep in mind that what we are doing
// here is for performance reasons only so we should only break if the real action (that is,
// the write) fails -- not any of the optimization steps.
currentContent = null;
}
if (currentContent == null) {
writeContent(outputFile, newContent);
} else {
if (!Arrays.equals(newContent, currentContent)) {
if (!outputFile.isWritable()) {
outputFile.delete();
}
writeContent(outputFile, newContent);
}
}
}
/**
* Returns the entirety of the specified input stream and returns it as a char
* array, decoding characters using ISO-8859-1 (Latin1).
*
* @throws IOException if there was an error
*/
public static char[] readContentAsLatin1(InputStream in) throws IOException {
return convertFromLatin1(ByteStreams.toByteArray(in));
}
/**
* Returns the entirety of the specified file and returns it as a char array,
* decoding characters using ISO-8859-1 (Latin1).
*
* @throws IOException if there was an error
*/
public static char[] readContentAsLatin1(Path inputFile) throws IOException {
return convertFromLatin1(readContent(inputFile));
}
/**
* Returns an iterable that allows iterating over ISO-8859-1 (Latin1) text
* file contents line by line. If the file ends in a line break, the iterator
* will return an empty string as the last element.
*
* @throws IOException if there was an error
*/
public static Iterable Use this method when you already know the size of the file. The sanity check is intended to
* catch issues where filesystems incorrectly truncate files.
*
* @throws IOException if there was an error, or if fewer than {@code fileSize} bytes were read.
*/
public static byte[] readWithKnownFileSize(Path path, long fileSize) throws IOException {
if (fileSize > Integer.MAX_VALUE) {
throw new IOException("Cannot read file with size larger than 2GB");
}
int fileSizeInt = (int) fileSize;
byte[] bytes = readContentWithLimit(path, fileSizeInt);
if (fileSizeInt > bytes.length) {
throw new IOException("Unexpected short read from file '" + path
+ "' (expected " + fileSizeInt + ", got " + bytes.length + " bytes)");
}
return bytes;
}
/**
* Returns the type of the file system path belongs to.
*/
public static String getFileSystem(Path path) {
return path.getFileSystem().getFileSystemType(path);
}
/**
* Returns whether the given path starts with any of the paths in the given
* list of prefixes.
*/
public static boolean startsWithAny(Path path, Iterable