diff options
18 files changed, 423 insertions, 300 deletions
diff --git a/scripts/bootstrap/compile.sh b/scripts/bootstrap/compile.sh index 25729a410a..1bb3a62e0a 100755 --- a/scripts/bootstrap/compile.sh +++ b/scripts/bootstrap/compile.sh @@ -18,7 +18,7 @@ PROTO_FILES=$(ls src/main/protobuf/*.proto) LIBRARY_JARS=$(find third_party -name '*.jar' | tr "\n" " ") -DIRS=$(echo src/{main/java,tools/xcode-common/java/com/google/devtools/build/xcode/{common,util}} ${OUTPUT_DIR}/src) +DIRS=$(echo src/{java_tools/singlejar/java/com/google/devtools/build/zip,main/java,tools/xcode-common/java/com/google/devtools/build/xcode/{common,util}} ${OUTPUT_DIR}/src) BLAZE_CC_FILES=( src/main/cpp/blaze_startup_options.cc diff --git a/src/java_tools/singlejar/BUILD b/src/java_tools/singlejar/BUILD index 2f9391db40..8d3685c7d2 100644 --- a/src/java_tools/singlejar/BUILD +++ b/src/java_tools/singlejar/BUILD @@ -1,5 +1,11 @@ package(default_visibility = ["//src:__subpackages__"]) +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src/test/shell/bazel:__pkg__"], +) + java_library( name = "libSingleJar", srcs = glob(["java/**/singlejar/**/*.java"]), diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipFileEntry.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipFileEntry.java index e8687f1c74..f4e08ab73c 100644 --- a/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipFileEntry.java +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipFileEntry.java @@ -437,4 +437,9 @@ public final class ZipFileEntry { EnumSet<Feature> getFeatureSet() { return featureSet; } + + @Override + public String toString() { + return "ZipFileEntry[" + name + "]"; + } } diff --git a/src/main/java/BUILD b/src/main/java/BUILD index 8b7ad922c6..0b1045c385 100644 --- a/src/main/java/BUILD +++ b/src/main/java/BUILD @@ -155,6 +155,7 @@ java_library( ":shell", ":unix", ":vfs", + "//src/java_tools/singlejar:zip", "//src/main/protobuf:proto_build", "//src/main/protobuf:proto_bundlemerge", "//src/main/protobuf:proto_crosstool_config", @@ -165,7 +166,6 @@ java_library( "//src/tools/xcode-common/java/com/google/devtools/build/xcode/common", "//src/tools/xcode-common/java/com/google/devtools/build/xcode/util", "//third_party:aether", - "//third_party:apache_commons_compress", "//third_party:apache_commons_pool2", "//third_party:auto_value", "//third_party:gson", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index c43801a102..f742683b1a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -25,12 +25,14 @@ import com.google.devtools.build.lib.bazel.commands.FetchCommand; import com.google.devtools.build.lib.bazel.repository.HttpArchiveFunction; import com.google.devtools.build.lib.bazel.repository.HttpDownloadFunction; import com.google.devtools.build.lib.bazel.repository.HttpJarFunction; +import com.google.devtools.build.lib.bazel.repository.JarFunction; import com.google.devtools.build.lib.bazel.repository.LocalRepositoryFunction; import com.google.devtools.build.lib.bazel.repository.MavenJarFunction; import com.google.devtools.build.lib.bazel.repository.NewHttpArchiveFunction; import com.google.devtools.build.lib.bazel.repository.NewLocalRepositoryFunction; import com.google.devtools.build.lib.bazel.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.bazel.repository.RepositoryFunction; +import com.google.devtools.build.lib.bazel.repository.ZipFunction; import com.google.devtools.build.lib.bazel.rules.android.AndroidNdkRepositoryFunction; import com.google.devtools.build.lib.bazel.rules.android.AndroidNdkRepositoryRule; import com.google.devtools.build.lib.bazel.rules.android.AndroidSdkRepositoryFunction; @@ -134,6 +136,8 @@ public class BazelRepositoryModule extends BlazeModule { // Helper SkyFunctions. builder.put(SkyFunctionName.computed(HttpDownloadFunction.NAME), new HttpDownloadFunction()); + builder.put(JarFunction.NAME, new JarFunction()); + builder.put(ZipFunction.NAME, new ZipFunction()); return builder.build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorFactory.java deleted file mode 100644 index d421b49ad9..0000000000 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorFactory.java +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2014 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.bazel.repository; - -import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule; -import com.google.devtools.build.lib.bazel.rules.workspace.HttpJarRule; -import com.google.devtools.build.lib.bazel.rules.workspace.NewHttpArchiveRule; -import com.google.devtools.build.lib.vfs.FileSystemUtils; -import com.google.devtools.build.lib.vfs.Path; -import com.google.devtools.build.lib.vfs.PathFragment; - -import org.apache.commons.compress.archivers.ArchiveException; -import org.apache.commons.compress.archivers.ArchiveInputStream; -import org.apache.commons.compress.archivers.ArchiveStreamFactory; -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.utils.IOUtils; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.Charset; - -/** - * Creates decompressors to use on archive. Use {@link DecompressorFactory#create} to get the - * correct type of decompressor for the input archive, then call - * {@link Decompressor#decompress} to decompress it. - */ -public abstract class DecompressorFactory { - - public static Decompressor create( - String targetKind, String targetName, Path archivePath, Path repositoryPath) - throws DecompressorException { - String baseName = archivePath.getBaseName(); - - if (targetKind.startsWith(HttpJarRule.NAME + " ")) { - if (baseName.endsWith(".jar")) { - return new JarDecompressor(targetKind, targetName, archivePath, repositoryPath); - } else { - throw new DecompressorException( - String.format("Expected %s %s to create file with a .jar suffix (got %s)", - HttpJarRule.NAME, targetName, archivePath)); - } - } - - if (targetKind.startsWith(HttpArchiveRule.NAME + " ") - || targetKind.startsWith(NewHttpArchiveRule.NAME + " ")) { - if (baseName.endsWith(".zip") || baseName.endsWith(".jar")) { - return new ZipDecompressor(archivePath); - } else { - throw new DecompressorException( - String.format("Expected %s %s to create file with a .zip or .jar suffix (got %s)", - HttpArchiveRule.NAME, targetName, archivePath)); - } - } - - throw new DecompressorException(String.format("No decompressor found for %s rule %s (got %s)", - targetKind, targetName, archivePath)); - } - - /** - * General decompressor for an archive. Should be overridden for each specific archive type. - */ - public abstract static class Decompressor { - protected final Path archiveFile; - - private Decompressor(Path archiveFile) { - this.archiveFile = archiveFile; - } - - /** - * This is overridden by archive-specific decompression logic. Often this logic will create - * files and directories under the {@link Decompressor#archiveFile}'s parent directory. - * - * @return the path to the repository directory. That is, the returned path will be a directory - * containing a WORKSPACE file. - */ - public abstract Path decompress() throws DecompressorException; - } - - /** - * Decompressor for jar files. - * - * <p>This is actually a bit of a misnomer, as .jars aren't decompressed. This does create a - * repository a BUILD file for them, though, making the java_import target @<jar>//jar:jar - * available for users to depend on.</p> - */ - static class JarDecompressor extends Decompressor { - private final String targetKind; - private final String targetName; - private final Path repositoryDir; - - public JarDecompressor( - String targetKind, String targetName, Path archiveFile, Path repositoryDir) { - super(archiveFile); - this.targetKind = targetKind; - this.targetName = targetName; - this.repositoryDir = repositoryDir; - } - - /** - * The .jar can be used compressed, so this just exposes it in a way Bazel can use. - * - * <p>It moves the jar from some-name/x/y/z/foo.jar to some-name/jar/foo.jar and creates a - * BUILD file containing one entry: the .jar. - */ - @Override - public Path decompress() throws DecompressorException { - // Example: archiveFile is .external-repository/some-name/foo.jar. - String baseName = archiveFile.getBaseName(); - - try { - FileSystemUtils.createDirectoryAndParents(repositoryDir); - // .external-repository/some-name/WORKSPACE. - Path workspaceFile = repositoryDir.getRelative("WORKSPACE"); - FileSystemUtils.writeContent(workspaceFile, Charset.forName("UTF-8"), String.format( - "# DO NOT EDIT: automatically generated WORKSPACE file for %s rule %s\n", - targetKind, targetName)); - // .external-repository/some-name/jar. - Path jarDirectory = repositoryDir.getRelative("jar"); - FileSystemUtils.createDirectoryAndParents(jarDirectory); - // .external-repository/some-name/repository/jar/foo.jar is a symbolic link to the jar in - // .external-repository/some-name. - Path jarSymlink = jarDirectory.getRelative(baseName); - if (!jarSymlink.exists()) { - jarSymlink.createSymbolicLink(archiveFile); - } - // .external-repository/some-name/repository/jar/BUILD defines the //jar target. - Path buildFile = jarDirectory.getRelative("BUILD"); - FileSystemUtils.writeLinesAs(buildFile, Charset.forName("UTF-8"), - "# DO NOT EDIT: automatically generated BUILD file for " + targetKind + " rule " - + targetName, - "java_import(", - " name = 'jar',", - " jars = ['" + baseName + "'],", - " visibility = ['//visibility:public']", - ")"); - } catch (IOException e) { - throw new DecompressorException( - "Error auto-creating jar repo structure: " + e.getMessage()); - } - return repositoryDir; - } - } - - /** - * Decompressor for zip files. - */ - private static class ZipDecompressor extends Decompressor { - public ZipDecompressor(Path archiveFile) { - super(archiveFile); - } - - /** - * This unzips the zip file to a sibling directory of {@link Decompressor#archiveFile}. The - * zip file is expected to have the WORKSPACE file at the top level, e.g.: - * - * <pre> - * $ unzip -lf some-repo.zip - * Archive: ../repo.zip - * Length Date Time Name - * --------- ---------- ----- ---- - * 0 2014-11-20 15:50 WORKSPACE - * 0 2014-11-20 16:10 foo/ - * 236 2014-11-20 15:52 foo/BUILD - * ... - * </pre> - */ - @Override - public Path decompress() throws DecompressorException { - Path destinationDirectory = archiveFile.getParentDirectory(); - try (InputStream is = new FileInputStream(archiveFile.getPathString())) { - ArchiveInputStream in = new ArchiveStreamFactory().createArchiveInputStream( - ArchiveStreamFactory.ZIP, is); - ZipArchiveEntry entry = (ZipArchiveEntry) in.getNextEntry(); - while (entry != null) { - extractZipEntry(in, entry, destinationDirectory); - entry = (ZipArchiveEntry) in.getNextEntry(); - } - } catch (IOException | ArchiveException e) { - throw new DecompressorException( - String.format("Error extracting %s to %s: %s", - archiveFile, destinationDirectory, e.getMessage())); - } - return destinationDirectory; - } - - private void extractZipEntry( - ArchiveInputStream in, ZipArchiveEntry entry, Path destinationDirectory) - throws IOException, DecompressorException { - PathFragment relativePath = new PathFragment(entry.getName()); - if (relativePath.isAbsolute()) { - throw new DecompressorException( - String.format("Failed to extract %s, zipped paths cannot be absolute", relativePath)); - } - Path outputPath = destinationDirectory.getRelative(relativePath); - FileSystemUtils.createDirectoryAndParents(outputPath.getParentDirectory()); - if (entry.isDirectory()) { - FileSystemUtils.createDirectoryAndParents(outputPath); - } else { - try (OutputStream out = new FileOutputStream(new File(outputPath.getPathString()))) { - IOUtils.copy(in, out); - } catch (IOException e) { - throw new DecompressorException( - String.format("Error writing %s from %s", outputPath, archiveFile)); - } - } - } - } - - /** - * Exceptions thrown when something goes wrong decompressing an archive. - */ - public static class DecompressorException extends Exception { - public DecompressorException(String message) { - super(message); - } - } -} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java new file mode 100644 index 0000000000..0ba8ef9797 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java @@ -0,0 +1,162 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule; +import com.google.devtools.build.lib.bazel.rules.workspace.HttpJarRule; +import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule; +import com.google.devtools.build.lib.bazel.rules.workspace.NewHttpArchiveRule; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.util.Objects; + +/** + * The contents of decompressed archive. + */ +public class DecompressorValue implements SkyValue { + + private final Path directory; + + /** + * @param repositoryPath + */ + public DecompressorValue(Path repositoryPath) { + directory = repositoryPath; + } + + public Path getDirectory() { + return directory; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || !(other instanceof DecompressorValue)) { + return false; + } + + return directory.equals(((DecompressorValue) other).directory); + } + + @Override + public int hashCode() { + return directory.hashCode(); + } + + public static SkyKey key( + String targetKind, String targetName, Path archivePath, Path repositoryPath) + throws IOException { + String baseName = archivePath.getBaseName(); + + if (targetKind.startsWith(HttpJarRule.NAME + " ") + || targetKind.equals(MavenJarRule.NAME)) { + if (baseName.endsWith(".jar")) { + return new SkyKey(JarFunction.NAME, + new DecompressorDescriptor(targetKind, targetName, archivePath, repositoryPath)); + } else { + throw new IOException( + String.format("Expected %s %s to create file with a .jar suffix (got %s)", + targetKind, targetName, archivePath)); + } + } + + if (targetKind.startsWith(HttpArchiveRule.NAME + " ") + || targetKind.startsWith(NewHttpArchiveRule.NAME + " ")) { + if (baseName.endsWith(".zip") || baseName.endsWith(".jar")) { + return new SkyKey(ZipFunction.NAME, + new DecompressorDescriptor(targetKind, targetName, archivePath, repositoryPath)); + } else { + throw new IOException( + String.format("Expected %s %s to create file with a .zip or .jar suffix (got %s)", + HttpArchiveRule.NAME, targetName, archivePath)); + } + } + + throw new IOException(String.format("No decompressor found for %s rule %s (got %s)", + targetKind, targetName, archivePath)); + } + + /** + * Description of an archive to be decompressed for use in a SkyKey. + * TODO(bazel-team): this should be an autovalue class. + */ + public static class DecompressorDescriptor { + private final String targetKind; + private final String targetName; + private final Path archivePath; + private final Path repositoryPath; + + public DecompressorDescriptor(String targetKind, String targetName, Path archivePath, + Path repositoryPath) { + this.targetKind = targetKind; + this.targetName = targetName; + this.archivePath = archivePath; + this.repositoryPath = repositoryPath; + } + + public String targetKind() { + return targetKind; + } + + public String targetName() { + return targetName; + } + + public Path archivePath() { + return archivePath; + } + + public Path repositoryPath() { + return repositoryPath; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || !(other instanceof DecompressorDescriptor)) { + return false; + } + + DecompressorDescriptor descriptor = (DecompressorDescriptor) other; + return targetKind.equals(descriptor.targetKind) + && targetName.equals(descriptor.targetName) + && archivePath.equals(descriptor.archivePath) + && repositoryPath.equals(descriptor.repositoryPath); + } + + @Override + public int hashCode() { + return Objects.hash(targetKind, targetName, archivePath, repositoryPath); + } + } + + /** + * Exceptions thrown when something goes wrong decompressing an archive. + */ + static class DecompressorException extends Exception { + public DecompressorException(String message) { + super(message); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java index 94bb25de15..b5cdb9bde6 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java @@ -15,15 +15,11 @@ package com.google.devtools.build.lib.bazel.repository; import com.google.devtools.build.lib.analysis.RuleDefinition; -import com.google.devtools.build.lib.bazel.repository.DecompressorFactory.DecompressorException; import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule; -import com.google.devtools.build.lib.packages.AggregatingAttributeMapper; import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName; import com.google.devtools.build.lib.packages.Rule; -import com.google.devtools.build.lib.packages.Type; import com.google.devtools.build.lib.skyframe.FileValue; import com.google.devtools.build.lib.skyframe.RepositoryValue; -import com.google.devtools.build.lib.syntax.EvalException; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunctionException; @@ -33,8 +29,6 @@ import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; /** * Downloads a file over HTTP. @@ -76,26 +70,23 @@ public class HttpArchiveFunction extends RepositoryFunction { if (directoryValue == null) { return null; } - AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule); - URL url = null; - try { - url = new URL(mapper.get("url", Type.STRING)); - } catch (MalformedURLException e) { - throw new RepositoryFunctionException( - new EvalException(rule.getLocation(), "Error parsing URL: " + e.getMessage()), - Transience.PERSISTENT); - } - String sha256 = mapper.get("sha256", Type.STRING); - HttpDownloader downloader = new HttpDownloader(url, sha256, outputDirectory); + try { - Path archiveFile = downloader.download(); - outputDirectory = DecompressorFactory.create( - rule.getTargetKind(), rule.getName(), archiveFile, outputDirectory).decompress(); + HttpDownloadValue downloadValue = (HttpDownloadValue) env.getValueOrThrow( + HttpDownloadFunction.key(rule, outputDirectory), IOException.class); + if (downloadValue == null) { + return null; + } + + DecompressorValue value = (DecompressorValue) env.getValueOrThrow(DecompressorValue.key( + rule.getTargetKind(), rule.getName(), downloadValue.getPath(), outputDirectory), + IOException.class); + if (value == null) { + return null; + } } catch (IOException e) { // Assumes all IO errors transient. throw new RepositoryFunctionException(e, Transience.TRANSIENT); - } catch (DecompressorException e) { - throw new RepositoryFunctionException(new IOException(e.getMessage()), Transience.TRANSIENT); } return RepositoryValue.create(directoryValue); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloadFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloadFunction.java index dc84864498..022d0dc000 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloadFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloadFunction.java @@ -101,7 +101,7 @@ public class HttpDownloadFunction implements SkyFunction { if (obj == this) { return true; } - if (!(obj instanceof HttpDescriptor)) { + if (obj == null || !(obj instanceof HttpDescriptor)) { return false; } HttpDescriptor other = (HttpDescriptor) obj; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloadValue.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloadValue.java index 39f3f24a4b..547e571720 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloadValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloadValue.java @@ -38,12 +38,11 @@ public class HttpDownloadValue implements SkyValue { if (this == other) { return true; } - - if (other instanceof HttpDownloadValue) { - HttpDownloadValue otherValue = (HttpDownloadValue) other; - return this.path.equals(otherValue.path); + if (other == null || !(other instanceof HttpDownloadValue)) { + return false; } - return false; + HttpDownloadValue otherValue = (HttpDownloadValue) other; + return this.path.equals(otherValue.path); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/JarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/JarFunction.java new file mode 100644 index 0000000000..e519a171db --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/JarFunction.java @@ -0,0 +1,91 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.google.devtools.build.lib.bazel.repository.DecompressorValue.DecompressorDescriptor; +import com.google.devtools.build.lib.bazel.repository.RepositoryFunction.RepositoryFunctionException; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.nio.charset.Charset; + +import javax.annotation.Nullable; + +/** + * Creates a repository for a jar file. + */ +public class JarFunction implements SkyFunction { + + public static final SkyFunctionName NAME = SkyFunctionName.computed("JAR_FUNCTION"); + + /** + * The .jar can be used compressed, so this just exposes it in a way Bazel can use. + * + * <p>It moves the jar from some-name/x/y/z/foo.jar to some-name/jar/foo.jar and creates a + * BUILD file containing one entry: the .jar.</p> + */ + @Override + @Nullable + public SkyValue compute(SkyKey skyKey, Environment env) throws RepositoryFunctionException { + DecompressorDescriptor descriptor = (DecompressorDescriptor) skyKey.argument(); + // Example: archiveFile is external/some-name/foo.jar. + String baseName = descriptor.archivePath().getBaseName(); + + try { + FileSystemUtils.createDirectoryAndParents(descriptor.repositoryPath()); + // external/some-name/WORKSPACE. + Path workspaceFile = descriptor.repositoryPath().getRelative("WORKSPACE"); + FileSystemUtils.writeContent(workspaceFile, Charset.forName("UTF-8"), String.format( + "# DO NOT EDIT: automatically generated WORKSPACE file for %s rule %s\n", + descriptor.targetKind(), descriptor.targetName())); + // external/some-name/jar. + Path jarDirectory = descriptor.repositoryPath().getRelative("jar"); + FileSystemUtils.createDirectoryAndParents(jarDirectory); + // external/some-name/repository/jar/foo.jar is a symbolic link to the jar in + // external/some-name. + Path jarSymlink = jarDirectory.getRelative(baseName); + if (!jarSymlink.exists()) { + jarSymlink.createSymbolicLink(descriptor.archivePath()); + } + // external/some-name/repository/jar/BUILD defines the //jar target. + Path buildFile = jarDirectory.getRelative("BUILD"); + FileSystemUtils.writeLinesAs(buildFile, Charset.forName("UTF-8"), + "# DO NOT EDIT: automatically generated BUILD file for " + descriptor.targetKind() + + " rule " + descriptor.targetName(), + "java_import(", + " name = 'jar',", + " jars = ['" + baseName + "'],", + " visibility = ['//visibility:public']", + ")"); + } catch (IOException e) { + throw new RepositoryFunctionException(new IOException( + "Error auto-creating jar repo structure: " + e.getMessage()), Transience.TRANSIENT); + } + return new DecompressorValue(descriptor.repositoryPath()); + } + + @Override + @Nullable + public String extractTag(SkyKey skyKey) { + return null; + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java index 8861beb8ab..8af654f7da 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java @@ -22,8 +22,6 @@ import com.google.common.collect.Lists; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; import com.google.devtools.build.lib.analysis.RuleDefinition; -import com.google.devtools.build.lib.bazel.repository.DecompressorFactory.DecompressorException; -import com.google.devtools.build.lib.bazel.repository.DecompressorFactory.JarDecompressor; import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule; import com.google.devtools.build.lib.packages.AggregatingAttributeMapper; import com.google.devtools.build.lib.packages.AttributeMap; @@ -88,10 +86,8 @@ public class MavenJarFunction extends HttpArchiveFunction { return downloader; } - @VisibleForTesting SkyValue createOutputTree(MavenDownloader downloader, Environment env) throws RepositoryFunctionException { - FileValue outputDirectoryValue = createDirectory(downloader.getOutputDirectory(), env); if (outputDirectoryValue == null) { return null; @@ -105,16 +101,18 @@ public class MavenJarFunction extends HttpArchiveFunction { } // Add a WORKSPACE file & BUILD file to the Maven jar. - JarDecompressor decompressor = new JarDecompressor( - MavenJarRule.NAME, downloader.getName(), repositoryJar, - outputDirectoryValue.realRootedPath().asPath()); - Path repositoryDirectory = null; + DecompressorValue value; try { - repositoryDirectory = decompressor.decompress(); - } catch (DecompressorException e) { - throw new RepositoryFunctionException(new IOException(e.getMessage()), Transience.TRANSIENT); + value = (DecompressorValue) env.getValueOrThrow(DecompressorValue.key( + MavenJarRule.NAME, downloader.getName(), repositoryJar, + outputDirectoryValue.realRootedPath().asPath()), IOException.class); + if (value == null) { + return null; + } + } catch (IOException e) { + throw new RepositoryFunctionException(e, Transience.TRANSIENT); } - FileValue repositoryFileValue = getRepositoryDirectory(repositoryDirectory, env); + FileValue repositoryFileValue = getRepositoryDirectory(value.getDirectory(), env); if (repositoryFileValue == null) { return null; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java index 3000a95150..30ad5c888a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java @@ -75,18 +75,21 @@ public class NewHttpArchiveFunction extends HttpArchiveFunction { } // Decompress. - Path decompressedDirectory; + DecompressorValue decompressed; try { - decompressedDirectory = DecompressorFactory.create( - rule.getTargetKind(), rule.getName(), downloadedFileValue.getPath(), outputDirectory) - .decompress(); - } catch (DecompressorFactory.DecompressorException e) { + decompressed = (DecompressorValue) env.getValueOrThrow( + DecompressorValue.key(rule.getTargetKind(), rule.getName(), + downloadedFileValue.getPath(), outputDirectory), IOException.class); + if (decompressed == null) { + return null; + } + } catch (IOException e) { throw new RepositoryFunctionException( new IOException(e.getMessage()), Transience.TRANSIENT); } // Add WORKSPACE and BUILD files. - createWorkspaceFile(decompressedDirectory, rule); + createWorkspaceFile(decompressed.getDirectory(), rule); return symlinkBuildFile(rule, getWorkspace(), repositoryDirectory, env); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryFunction.java index efc2beb7d2..b4a02a2a80 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryFunction.java @@ -171,11 +171,6 @@ public abstract class RepositoryFunction implements SkyFunction { if (createSymbolicLink(buildFilePath, buildFileTarget, env) == null) { return null; } - - if (buildFileValue == null) { - return null; - } - return RepositoryValue.createNew(directoryValue, buildFileValue); } @@ -201,7 +196,7 @@ public abstract class RepositoryFunction implements SkyFunction { * .external-repository/ * x/ * WORKSPACE - * BUILD -> <build_root>/x.BUILD + * BUILD -> <build_root>/x.BUILD * z -> /some/path/to/y/z * w -> /some/path/to/y/w * v -> /some/path/to/y/v diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/ZipFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/ZipFunction.java new file mode 100644 index 0000000000..168220cb8f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/ZipFunction.java @@ -0,0 +1,106 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.google.devtools.build.lib.bazel.repository.DecompressorValue.DecompressorDescriptor; +import com.google.devtools.build.lib.bazel.repository.RepositoryFunction.RepositoryFunctionException; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.build.zip.ZipFileEntry; +import com.google.devtools.build.zip.ZipReader; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Collection; + +import javax.annotation.Nullable; + +/** + * Creates a repository by decompressing a zip file. + */ +public class ZipFunction implements SkyFunction { + + public static final SkyFunctionName NAME = SkyFunctionName.computed("ZIP_FUNCTION"); + + /** + * This unzips the zip file to a sibling directory of {@link DecompressorDescriptor#archivePath}. + * The zip file is expected to have the WORKSPACE file at the top level, e.g.: + * + * <pre> + * $ unzip -lf some-repo.zip + * Archive: ../repo.zip + * Length Date Time Name + * --------- ---------- ----- ---- + * 0 2014-11-20 15:50 WORKSPACE + * 0 2014-11-20 16:10 foo/ + * 236 2014-11-20 15:52 foo/BUILD + * ... + * </pre> + */ + @Override + @Nullable + public SkyValue compute(SkyKey skyKey, Environment env) throws RepositoryFunctionException { + DecompressorDescriptor descriptor = (DecompressorDescriptor) skyKey.argument(); + Path destinationDirectory = descriptor.archivePath().getParentDirectory(); + try (ZipReader reader = new ZipReader(descriptor.archivePath().getPathFile())) { + Collection<ZipFileEntry> entries = reader.entries(); + for (ZipFileEntry entry : entries) { + extractZipEntry(reader, entry, destinationDirectory); + } + } catch (IOException e) { + throw new RepositoryFunctionException(new IOException( + String.format("Error extracting %s to %s: %s", + descriptor.archivePath(), destinationDirectory, e.getMessage())), + Transience.TRANSIENT); + } + return new DecompressorValue(destinationDirectory); + } + + private void extractZipEntry(ZipReader reader, ZipFileEntry entry, Path destinationDirectory) + throws IOException { + PathFragment relativePath = new PathFragment(entry.getName()); + if (relativePath.isAbsolute()) { + throw new IOException( + String.format("Failed to extract %s, zipped paths cannot be absolute", relativePath)); + } + Path outputPath = destinationDirectory.getRelative(relativePath); + FileSystemUtils.createDirectoryAndParents(outputPath.getParentDirectory()); + // Posix permissions are in the high-order 2 bytes of the external attributes. After this + // operation, permissions holds 0100755 (or 040755 for directories). + int permissions = entry.getExternalAttributes() >>> 16; + boolean isDirectory = (permissions & 040000) == 040000; + if (isDirectory) { + FileSystemUtils.createDirectoryAndParents(outputPath); + } else { + File outputFile = outputPath.getPathFile(); + Files.copy(reader.getInputStream(entry), outputFile.toPath()); + outputPath.chmod(permissions); + } + } + + @Override + @Nullable + public String extractTag(SkyKey skyKey) { + return null; + } + +} diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index 9b976d9d8a..12a9bdd259 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -52,6 +52,7 @@ genrule( name = "doc-srcs", testonly = 1, srcs = [ + "//src/java_tools/singlejar:srcs", "//src/main/protobuf:srcs", "//src/main/java:srcs", "//src/tools/xcode-common:srcs", diff --git a/src/test/shell/bazel/external_integration_test.sh b/src/test/shell/bazel/external_integration_test.sh index e6c02ea455..32757b7503 100755 --- a/src/test/shell/bazel/external_integration_test.sh +++ b/src/test/shell/bazel/external_integration_test.sh @@ -141,7 +141,11 @@ filegroup( ) EOF what_does_the_fox_say="Fraka-kaka-kaka-kaka-kow" - echo $what_does_the_fox_say > fox/male + cat > fox/male <<EOF +#!/bin/bash +echo $what_does_the_fox_say +EOF + chmod +x fox/male # Add some padding to the .zip to test that Bazel's download logic can # handle breaking a response into chunks. dd if=/dev/zero of=fox/padding bs=1024 count=10240 @@ -166,11 +170,10 @@ EOF cat > zoo/female.sh <<EOF #!/bin/bash -cat external/endangered/fox/male +./external/endangered/fox/male EOF chmod +x zoo/female.sh - bazel fetch //zoo:breeding-program || fail "Fetch failed" bazel run //zoo:breeding-program >& $TEST_log \ || echo "Expected build/run to succeed" kill_nc @@ -403,7 +406,7 @@ function test_new_remote_repo() { local what_does_the_fox_say="Fraka-kaka-kaka-kaka-kow" echo $what_does_the_fox_say > fox/male local repo2_zip=$TEST_TMPDIR/fox.zip - rm $repo2_zip + rm -f $repo2_zip zip -r $repo2_zip fox local sha256=$(sha256sum $repo2_zip | cut -f 1 -d ' ') serve_file $repo2_zip @@ -424,7 +427,6 @@ new_http_archive( sha256 = '$sha256', build_file = 'fox.BUILD' ) -bind(name = 'stud', actual = '@endangered//:fox') EOF mkdir -p zoo @@ -432,7 +434,7 @@ EOF sh_binary( name = "breeding-program", srcs = ["female.sh"], - data = ["//external:stud"], + data = ["@endangered//:fox"], ) EOF @@ -442,8 +444,6 @@ cat external/endangered/fox/male EOF chmod +x zoo/female.sh - bazel clean --expunge - bazel fetch //zoo:breeding-program || fail "Fetch failed" bazel run //zoo:breeding-program >& $TEST_log \ || echo "Expected build/run to succeed" kill_nc diff --git a/third_party/BUILD b/third_party/BUILD index 146f63dd4e..0f11d72beb 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -47,11 +47,6 @@ java_import( ) java_import( - name = "apache_commons_compress", - jars = ["apache_commons_compress/apache-commons-compress-1.9.jar"], -) - -java_import( name = "apache_commons_logging", jars = ["apache_commons_logging/commons-logging-1.1.1.jar"], ) |