From 5aa12f1089bd9eff8ef95ebfc5351eb79018adad Mon Sep 17 00:00:00 2001 From: Kristina Chodorow Date: Tue, 21 Jul 2015 13:46:17 +0000 Subject: Use Apache Commons for reading tar files Fixes #305. -- MOS_MIGRATED_REVID=98730375 --- src/main/java/BUILD | 1 + .../build/lib/bazel/repository/TarGzFunction.java | 187 ++------------------- 2 files changed, 11 insertions(+), 177 deletions(-) diff --git a/src/main/java/BUILD b/src/main/java/BUILD index a8252ab8eb..25aa0edb5e 100644 --- a/src/main/java/BUILD +++ b/src/main/java/BUILD @@ -451,6 +451,7 @@ java_library( "//src/main/protobuf:proto_test_status", "//src/main/protobuf:proto_worker_protocol", "//third_party:aether", + "//third_party:apache_commons_compress", "//third_party:apache_commons_pool2", "//third_party:auto_value", "//third_party:guava", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/TarGzFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/TarGzFunction.java index 85100001f9..81be03d587 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/TarGzFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/TarGzFunction.java @@ -14,7 +14,6 @@ package com.google.devtools.build.lib.bazel.repository; -import com.google.common.io.CountingInputStream; import com.google.devtools.build.lib.bazel.repository.DecompressorValue.DecompressorDescriptor; import com.google.devtools.build.lib.bazel.repository.RepositoryFunction.RepositoryFunctionException; import com.google.devtools.build.lib.vfs.FileSystemUtils; @@ -22,12 +21,15 @@ import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyFunctionName; + import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; + import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.nio.file.Files; import java.util.zip.GZIPInputStream; @@ -47,16 +49,16 @@ public class TarGzFunction implements SkyFunction { try (GZIPInputStream gzipStream = new GZIPInputStream( new FileInputStream(descriptor.archivePath().getPathFile()))) { - TarInputStream inputStream = new TarInputStream(gzipStream); - while (inputStream.available() != 0) { - TarEntry entry = inputStream.getNextEntry(); - Path filename = descriptor.repositoryPath().getRelative(entry.getFilename()); + TarArchiveInputStream tarStream = new TarArchiveInputStream(gzipStream); + TarArchiveEntry entry; + while ((entry = tarStream.getNextTarEntry()) != null) { + Path filename = descriptor.repositoryPath().getRelative(entry.getName()); FileSystemUtils.createDirectoryAndParents(filename.getParentDirectory()); if (entry.isDirectory()) { FileSystemUtils.createDirectoryAndParents(filename); } else { - Files.copy(entry, filename.getPathFile().toPath()); - filename.chmod(entry.getPermissions()); + Files.copy(tarStream, filename.getPathFile().toPath()); + filename.chmod(entry.getMode()); } } } catch (IOException e) { @@ -71,173 +73,4 @@ public class TarGzFunction implements SkyFunction { return null; } - private static class TarInputStream { - private final CountingInputStream inputStream; - private String nextFilename; - - public TarInputStream(InputStream inputStream) { - this.inputStream = new CountingInputStream(inputStream); - } - - public int available() throws IOException { - nextFilename = TarEntry.parseFilename(inputStream); - if (nextFilename.isEmpty()) { - // We've probably reached the padding at the end of the .tar file. - return 0; - } - return inputStream.available(); - } - - public TarEntry getNextEntry() throws IOException { - return new TarEntry(nextFilename, inputStream); - } - } - - private static class TarEntry extends InputStream { - private static final int BUFFER_SIZE = 512; - private static final int FILENAME_SIZE = 100; - private static final int PERMISSIONS_SIZE = 8; - private static final int FILE_SIZE = 12; - private static final int TYPE_SIZE = 1; - private static final int USTAR_SIZE = 6; - - public enum FileType { - NORMAL, DIRECTORY - } - - private final String filename; - private final int permissions; - private final FileType type; - private final CountingInputStream inputStream; - // Tar format pads the content to blocks of 512 bytes, so when we're done reading the file skip - // this many bytes to arrive at the next file. - private final int finalSkip; - private long bytesRemaining; - private boolean done; - - public TarEntry(String filename, CountingInputStream inputStream) throws IOException { - byte buffer[] = new byte[BUFFER_SIZE]; - - this.filename = filename; - - // Permissions. - if (inputStream.read(buffer, 0, PERMISSIONS_SIZE) != PERMISSIONS_SIZE) { - throw new IOException("Error reading tar file (could not read permissions for " + filename - + ")"); - } - - String permissionsString; - if (buffer[PERMISSIONS_SIZE - 2] == ' ') { - // The permissions look like 000644 \0 (OS X, sigh). - permissionsString = new String(buffer, 0, PERMISSIONS_SIZE - 2); - } else { - // The permissions look like 0000644\0 (Linux). - permissionsString = new String(buffer, 0, PERMISSIONS_SIZE - 1); - } - try { - permissions = Integer.parseInt(permissionsString, 8); - } catch (NumberFormatException e) { - throw new IOException("Error reading tar file (could not parse permissions of " + filename - + "): [" + permissionsString + "]"); - } - - // User & group IDs. - inputStream.skip(16); - - // File size. - if (inputStream.read(buffer, 0, FILE_SIZE) != FILE_SIZE) { - throw new IOException("Error reading tar file (could not read file size of " + filename - + ")"); - } - - // 12345678901\0 in base 8, bizarly. - bytesRemaining = Long.parseLong(new String(buffer, 0, FILE_SIZE - 1), 8); - if (bytesRemaining % 512 == 0) { - if (bytesRemaining == 0) { - done = true; - } - finalSkip = 0; - } else { - done = false; - finalSkip = (int) (512 - bytesRemaining % 512); - } - - // Timestamp and checksum. - // TODO(kchodorow): actually check the checksum. - inputStream.skip(20); - - if (inputStream.read(buffer, 0, TYPE_SIZE) != TYPE_SIZE) { - throw new IOException("Error reading tar file (could not read file type of " + filename - + ")"); - } - char type = (char) buffer[0]; - if (type == '0') { - this.type = FileType.NORMAL; - } else if (type == '5') { - this.type = FileType.DIRECTORY; - } else { - // TODO(kchodorow): support links. - throw new IOException("Error reading tar file (unknown file type " + type + " for file " - + filename + ")"); - } - - // Skip name of linked file. - inputStream.skip(100); - - // USTAR constant. - if (inputStream.read(buffer, 0, USTAR_SIZE) != USTAR_SIZE - || !new String(buffer, 0, USTAR_SIZE - 1).equals("ustar")) { - // TODO(kchodorow): support old-style tar format. - throw new IOException("Error reading tar file (" + filename + " did not specify 'ustar')"); - } - - // Skip the rest of the ustar preamble. - inputStream.skip(249); - // We're now at position 512. - - // Ready to read content. - this.inputStream = inputStream; - } - - private static String parseFilename(InputStream inputStream) throws IOException { - byte buffer[] = new byte[FILENAME_SIZE]; - - if (inputStream.read(buffer, 0, FILENAME_SIZE) != FILENAME_SIZE) { - throw new IOException("Error reading tar file (could not read filename)"); - } - - int actualFilenameLength = 0; - while (actualFilenameLength < FILENAME_SIZE) { - if (buffer[actualFilenameLength] == 0) { - break; - } - actualFilenameLength++; - } - return new String(buffer, 0, actualFilenameLength); - } - - public String getFilename() { - return filename; - } - - public int getPermissions() { - return permissions; - } - - public boolean isDirectory() { - return type == FileType.DIRECTORY; - } - - @Override - public int read() throws IOException { - if (--bytesRemaining < 0) { - if (!done) { - inputStream.skip(finalSkip); - } - done = true; - return -1; - } - return inputStream.read(); - } - } } -- cgit v1.2.3