diff options
Diffstat (limited to 'src/java_tools/singlejar/java/com')
12 files changed, 2722 insertions, 0 deletions
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ConcatenateStrategy.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ConcatenateStrategy.java new file mode 100644 index 0000000000..7dfb31fb1c --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ConcatenateStrategy.java @@ -0,0 +1,74 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.singlejar; + +import com.google.devtools.build.singlejar.ZipEntryFilter.CustomMergeStrategy; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import javax.annotation.concurrent.NotThreadSafe; + +/** + * A strategy that merges a set of files by concatenating them. This is used + * for services files. By default, this class automatically adds a newline + * character {@code '\n'} between files if the previous file did not end with one. + * + * <p>Note: automatically inserting newline characters differs from the + * original behavior. Use {@link #ConcatenateStrategy(boolean)} to turn this + * behavior off. + */ +@NotThreadSafe +public final class ConcatenateStrategy implements CustomMergeStrategy { + + // The strategy assumes that files are generally small. This is a first guess + // about the size of the files. + private static final int BUFFER_SIZE = 4096; + + private final byte[] buffer = new byte[BUFFER_SIZE]; + private byte lastByteCopied = '\n'; + private final boolean appendNewLine; + + ConcatenateStrategy() { + this(true); + } + + /** + * @param appendNewLine Whether to add a newline character between files if + * the previous file did not end with one. + */ + ConcatenateStrategy(boolean appendNewLine) { + this.appendNewLine = appendNewLine; + } + + @Override + public void merge(InputStream in, OutputStream out) throws IOException { + if (appendNewLine && lastByteCopied != '\n') { + out.write('\n'); + lastByteCopied = '\n'; + } + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + lastByteCopied = buffer[bytesRead - 1]; + } + } + + @Override + public void finish(OutputStream out) { + // No need to do anything. All the data was already written. + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/CopyEntryFilter.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/CopyEntryFilter.java new file mode 100644 index 0000000000..586c378d64 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/CopyEntryFilter.java @@ -0,0 +1,33 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.singlejar; + +import java.io.IOException; + +import javax.annotation.concurrent.Immutable; + +/** + * A filter which invokes {@link StrategyCallback#copy} for every entry. As a + * result, the first entry for every given name is copied and further entries + * with the same name are skipped. + */ +@Immutable +public final class CopyEntryFilter implements ZipEntryFilter { + + @Override + public void accept(String filename, StrategyCallback callback) throws IOException { + callback.copy(null); + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/DefaultJarEntryFilter.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/DefaultJarEntryFilter.java new file mode 100644 index 0000000000..fd26c60986 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/DefaultJarEntryFilter.java @@ -0,0 +1,119 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.singlejar; + +import java.io.IOException; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.jar.JarFile; + +import javax.annotation.concurrent.Immutable; + +/** + * A default filter for JAR files. It merges all services files in the {@code META-INF/services/} + * directory. The original {@code MANIFEST} files are skipped, as are JAR signing files. Anything + * not in the supplied path filter, an arbitrary predicate, is also skipped. To use this filter + * properly, a new {@code MANIFEST} file should be explicitly added to the combined ZIP file. + */ +@Immutable +public class DefaultJarEntryFilter implements ZipEntryFilter { + + /** An interface to restrict which files are copied over and which are not. */ + public static interface PathFilter { + /** + * Returns true if an entry with the given name may be copied over. + */ + boolean allowed(String path); + } + + /** A filter that allows any path. */ + public static final PathFilter ANY_PATH = new PathFilter() { + @Override + public boolean allowed(String path) { + return true; + } + }; + + // ZIP timestamps have a resolution of 2 seconds, so this is the next timestamp after 1/1/1980. + // This is only Visible for testing. + static final Date DOS_EPOCH_PLUS_2_SECONDS = + new GregorianCalendar(1980, 0, 1, 0, 0, 2).getTime(); + + // Merge all files with a name in here: + private static final String SERVICES_DIR = "META-INF/services/"; + + // Merge all spring.handlers files. + private static final String SPRING_HANDLERS = "META-INF/spring.handlers"; + + // Merge all spring.schemas files. + private static final String SPRING_SCHEMAS = "META-INF/spring.schemas"; + + // Ignore all files with this name: + private static final String MANIFEST_NAME = JarFile.MANIFEST_NAME; + + // Merge all protobuf extension registries. + private static final String PROTOBUF_META = "protobuf.meta"; + + protected final Date date; + protected final Date classDate; + protected PathFilter allowedPaths; + + public DefaultJarEntryFilter(boolean normalize, PathFilter allowedPaths) { + this.date = normalize ? ZipCombiner.DOS_EPOCH : null; + this.classDate = normalize ? DOS_EPOCH_PLUS_2_SECONDS : null; + this.allowedPaths = allowedPaths; + } + + public DefaultJarEntryFilter(boolean normalize) { + this(normalize, ANY_PATH); + } + + public DefaultJarEntryFilter() { + this(true); + } + + @Override + public void accept(String filename, StrategyCallback callback) throws IOException { + if (!allowedPaths.allowed(filename)) { + callback.skip(); + } else if (filename.equals(SPRING_HANDLERS)) { + callback.customMerge(date, new ConcatenateStrategy()); + } else if (filename.equals(SPRING_SCHEMAS)) { + callback.customMerge(date, new ConcatenateStrategy()); + } else if (filename.startsWith(SERVICES_DIR)) { + // Merge all services files. + callback.customMerge(date, new ConcatenateStrategy()); + } else if (filename.equals(MANIFEST_NAME) || filename.endsWith(".SF") + || filename.endsWith(".DSA") || filename.endsWith(".RSA")) { + // Ignore existing manifests and any .SF, .DSA or .RSA jar signing files. + // TODO(bazel-team): I think we should be stricter and only skip signing + // files from the META-INF/ directory. + callback.skip(); + } else if (filename.endsWith(".class")) { + // Copy .class files over, but 2 seconds ahead of the dos epoch. If it finds both source and + // class files on the classpath, javac prefers the source file, if the class file is not newer + // than the source file. Since we normalize the timestamps, we need to provide timestamps for + // class files that are newer than those for the corresponding source files. + callback.copy(classDate); + } else if (filename.equals(PROTOBUF_META)) { + // Merge all protobuf meta data without inserting newlines, + // since the file is in protobuf binary format. + callback.customMerge(date, new ConcatenateStrategy(false)); + } else { + // Copy all other files over. + callback.copy(date); + } + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ExtraData.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ExtraData.java new file mode 100644 index 0000000000..2e8cb75e02 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ExtraData.java @@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.singlejar; + +/** + * A holder class for extra data in a ZIP entry. + * + * <p>Note: This class performs no defensive copying of the byte array, so the + * byte array passed into this class or returned from this class may not be + * modified. + */ +final class ExtraData { + + private final short id; + private final byte[] data; + + public ExtraData(short id, byte[] data) { + this.id = id; + this.data = data; + } + + public short getId() { + return id; + } + + public byte[] getData() { + return data; + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JarUtils.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JarUtils.java new file mode 100644 index 0000000000..a9c8ee3f65 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JarUtils.java @@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.singlejar; + +import java.io.IOException; +import java.util.Date; + +/** + * Provides utilities for using ZipCombiner to pack up Jar files. + */ +public final class JarUtils { + private static final String MANIFEST_DIRECTORY = "META-INF/"; + private static final short MAGIC_JAR_ID = (short) 0xCAFE; + private static final ExtraData[] MAGIC_JAR_ID_EXTRA_ENTRIES = + new ExtraData[] { new ExtraData(MAGIC_JAR_ID, new byte[0]) }; + + /** + * Adds META-INF directory through ZipCombiner with the given date and the + * magic jar ID. + * + * @throws IOException if {@link ZipCombiner#addDirectory(String, Date, ExtraData[])} + * throws an IOException. + */ + public static void addMetaInf(ZipCombiner combiner, Date date) throws IOException { + combiner.addDirectory(MANIFEST_DIRECTORY, date, MAGIC_JAR_ID_EXTRA_ENTRIES); + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JavaIoFileSystem.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JavaIoFileSystem.java new file mode 100644 index 0000000000..0da6e33040 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JavaIoFileSystem.java @@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.singlejar; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * An implementation based on java.io. + */ +public final class JavaIoFileSystem implements SimpleFileSystem { + + @Override + public InputStream getInputStream(String filename) throws IOException { + return new FileInputStream(filename); + } + + @Override + public OutputStream getOutputStream(String filename) throws IOException { + return new FileOutputStream(filename); + } + + @Override + public boolean delete(String filename) { + return new File(filename).delete(); + } +}
\ No newline at end of file diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/OptionFileExpander.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/OptionFileExpander.java new file mode 100644 index 0000000000..dafeb9d1d4 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/OptionFileExpander.java @@ -0,0 +1,127 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.singlejar; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.devtools.build.lib.shell.ShellUtils; +import com.google.devtools.build.lib.shell.ShellUtils.TokenizationException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; + +import javax.annotation.concurrent.Immutable; + +/** + * A utility class to parse option files and expand them. + */ +@Immutable +final class OptionFileExpander { + + /** + * An interface that allows injecting different implementations for reading + * files. This is mostly used for testing. + */ + interface OptionFileProvider { + + /** + * Opens a file for reading and returns an input stream. + */ + InputStream getInputStream(String filename) throws IOException; + } + + private final OptionFileProvider fileSystem; + + /** + * Creates an instance with the given option file provider. + */ + public OptionFileExpander(OptionFileProvider fileSystem) { + this.fileSystem = fileSystem; + } + + /** + * Pre-processes an argument list, expanding options of the form &at;filename + * to read in the content of the file and add it to the list of arguments. + * + * @param args the List of arguments to pre-process. + * @return the List of pre-processed arguments. + * @throws IOException if one of the files containing options cannot be read. + */ + public List<String> expandArguments(List<String> args) throws IOException { + List<String> expanded = new ArrayList<>(args.size()); + for (String arg : args) { + expandArgument(arg, expanded); + } + return expanded; + } + + /** + * Expands a single argument, expanding options &at;filename to read in + * the content of the file and add it to the list of processed arguments. + * + * @param arg the argument to pre-process. + * @param expanded the List of pre-processed arguments. + * @throws IOException if one of the files containing options cannot be read. + */ + private void expandArgument(String arg, List<String> expanded) throws IOException { + if (arg.startsWith("@")) { + InputStream in = fileSystem.getInputStream(arg.substring(1)); + try { + // TODO(bazel-team): This code doesn't handle escaped newlines correctly. + // ShellUtils doesn't support them either. + for (String line : readAllLines(new InputStreamReader(in, ISO_8859_1))) { + List<String> parsedTokens = new ArrayList<>(); + try { + ShellUtils.tokenize(parsedTokens, line); + } catch (TokenizationException e) { + throw new IOException("Could not tokenize parameter file!", e); + } + for (String token : parsedTokens) { + expandArgument(token, expanded); + } + } + InputStream inToClose = in; + in = null; + inToClose.close(); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + // Ignore the exception. It can only occur if an exception already + // happened and in that case, we want to preserve the original one. + } + } + } + } else { + expanded.add(arg); + } + } + + private List<String> readAllLines(Reader in) throws IOException { + List<String> result = new ArrayList<>(); + BufferedReader reader = new BufferedReader(in); + String line; + while ((line = reader.readLine()) != null) { + result.add(line); + } + return result; + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/PrefixListPathFilter.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/PrefixListPathFilter.java new file mode 100644 index 0000000000..a6e30d4478 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/PrefixListPathFilter.java @@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.singlejar; + +import com.google.devtools.build.singlejar.DefaultJarEntryFilter.PathFilter; + +import java.util.List; + +/** + * A predicate used to filter jar entries according to a list of path prefixes. + */ +final class PrefixListPathFilter implements PathFilter { + private final List<String> prefixes; + + public PrefixListPathFilter(List<String> prefixes) { + this.prefixes = prefixes; + } + + @Override + public boolean allowed(String path) { + for (String prefix : prefixes) { + if (path.startsWith(prefix)) { + return true; + } + } + return false; + } +}
\ No newline at end of file diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SimpleFileSystem.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SimpleFileSystem.java new file mode 100644 index 0000000000..844f12b714 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SimpleFileSystem.java @@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.singlejar; + +import com.google.devtools.build.singlejar.OptionFileExpander.OptionFileProvider; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * A simple virtual file system interface. It's much simpler than the Blaze + * virtual file system and only to be used inside this package. + */ +public interface SimpleFileSystem extends OptionFileProvider { + + @Override + InputStream getInputStream(String filename) throws IOException; + + /** + * Opens a file for output and returns an output stream. If a file of that + * name already exists, it is overwritten. + */ + OutputStream getOutputStream(String filename) throws IOException; + + /** + * Delete the file with the given name and return whether deleting it was + * successfull. + */ + boolean delete(String filename); +}
\ No newline at end of file diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SingleJar.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SingleJar.java new file mode 100644 index 0000000000..4551fd1813 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SingleJar.java @@ -0,0 +1,401 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.singlejar; + +import com.google.devtools.build.singlejar.DefaultJarEntryFilter.PathFilter; +import com.google.devtools.build.singlejar.ZipCombiner.OutputMode; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; +import java.util.jar.Attributes; +import java.util.jar.JarFile; +import java.util.jar.Manifest; + +import javax.annotation.concurrent.NotThreadSafe; + +/** + * An application that emulates the existing SingleJar tool, using the {@link + * ZipCombiner} class. + */ +@NotThreadSafe +public class SingleJar { + + private static final byte NEWLINE_BYTE = (byte) '\n'; + private static final String MANIFEST_FILENAME = JarFile.MANIFEST_NAME; + private static final String BUILD_DATA_FILENAME = "build-data.properties"; + + private final SimpleFileSystem fileSystem; + + /** The input jar files we want to combine into the output jar. */ + private final List<String> inputJars = new ArrayList<>(); + + /** Additional resources to be added to the output jar. */ + private final List<String> resources = new ArrayList<>(); + + /** Additional class path resources to be added to the output jar. */ + private final List<String> classpathResources = new ArrayList<>(); + + /** The name of the output Jar file. */ + private String outputJar; + + /** A filter for what jar entries to include */ + private PathFilter allowedPaths = DefaultJarEntryFilter.ANY_PATH; + + /** Extra manifest contents. */ + private String extraManifestContent; + /** The main class - this is put into the manifest and also into the build info. */ + private String mainClass; + + /** + * Warn about duplicate resource files, and skip them. Default behavior is to + * give an error message. + */ + private boolean warnDuplicateFiles = false; + + /** Indicates whether to set all timestamps to a fixed value. */ + private boolean normalize = false; + private OutputMode outputMode = OutputMode.FORCE_STORED; + + /** Whether to include build-data.properties file */ + protected boolean includeBuildData = true; + + /** List of build information properties files */ + protected List<String> buildInformationFiles = new ArrayList<String>(); + + /** Extraneous build informations (key=value) */ + protected List<String> buildInformations = new ArrayList<String>(); + + /** The (optional) native executable that will be prepended to this JAR. */ + private String launcherBin = null; + + // Only visible for testing. + protected SingleJar(SimpleFileSystem fileSystem) { + this.fileSystem = fileSystem; + } + + /** + * Creates a manifest and returns an input stream for its contents. + */ + private InputStream createManifest() throws IOException { + Manifest manifest = new Manifest(); + Attributes attributes = manifest.getMainAttributes(); + attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); + attributes.put(new Attributes.Name("Created-By"), "blaze-singlejar"); + if (mainClass != null) { + attributes.put(Attributes.Name.MAIN_CLASS, mainClass); + } + if (extraManifestContent != null) { + ByteArrayInputStream in = new ByteArrayInputStream(extraManifestContent.getBytes("UTF8")); + manifest.read(in); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + manifest.write(out); + return new ByteArrayInputStream(out.toByteArray()); + } + + private InputStream createBuildData() throws IOException { + Properties properties = mergeBuildData(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + properties.store(outputStream, null); + byte[] output = outputStream.toByteArray(); + // Properties#store() adds a timestamp comment as first line, delete it. + output = stripFirstLine(output); + return new ByteArrayInputStream(output); + } + + static byte[] stripFirstLine(byte[] output) { + int i = 0; + while (i < output.length && output[i] != NEWLINE_BYTE) { + i++; + } + if (i < output.length) { + output = Arrays.copyOfRange(output, i + 1, output.length); + } else { + output = new byte[0]; + } + return output; + } + + private Properties mergeBuildData() throws IOException { + Properties properties = new Properties(); + for (String fileName : buildInformationFiles) { + InputStream file = fileSystem.getInputStream(fileName); + if (file != null) { + properties.load(file); + } + } + + // extra properties + for (String info : buildInformations) { + String[] split = info.split("=", 2); + String key = split[0]; + String value = ""; + if (split.length > 1) { + value = split[1]; + } + properties.put(key, value); + } + + // finally add generic information + // TODO(bazel-team) do we need to resolve the path to be absolute or canonical? + properties.put("build.target", outputJar); + if (mainClass != null) { + properties.put("main.class", mainClass); + } + return properties; + } + + private String getName(String filename) { + int index = filename.lastIndexOf('/'); + return index < 0 ? filename : filename.substring(index + 1); + } + + // Only visible for testing. + protected int run(List<String> args) throws IOException { + List<String> expandedArgs = new OptionFileExpander(fileSystem).expandArguments(args); + processCommandlineArgs(expandedArgs); + InputStream buildInfo = createBuildData(); + + ZipCombiner combiner = null; + try { + combiner = new ZipCombiner(outputMode, createEntryFilter(normalize, allowedPaths), + fileSystem.getOutputStream(outputJar)); + if (launcherBin != null) { + combiner.prependExecutable(fileSystem.getInputStream(launcherBin)); + } + Date date = normalize ? ZipCombiner.DOS_EPOCH : null; + + // Add a manifest file. + JarUtils.addMetaInf(combiner, date); + combiner.addFile(MANIFEST_FILENAME, date, createManifest()); + + if (includeBuildData) { + // Add the build data file. + combiner.addFile(BUILD_DATA_FILENAME, date, buildInfo); + } + + // Copy the resources to the top level of the jar file. + for (String classpathResource : classpathResources) { + String entryName = getName(classpathResource); + if (warnDuplicateFiles && combiner.containsFile(entryName)) { + System.err.println("File " + entryName + " clashes with a previous file"); + continue; + } + combiner.addFile(entryName, date, fileSystem.getInputStream(classpathResource)); + } + + // Copy the resources into the jar file. + for (String resource : resources) { + String from, to; + int i = resource.indexOf(':'); + if (i < 0) { + to = from = resource; + } else { + from = resource.substring(0, i); + to = resource.substring(i + 1); + } + if (warnDuplicateFiles && combiner.containsFile(to)) { + System.err.println("File " + from + " at " + to + " clashes with a previous file"); + continue; + } + combiner.addFile(to, date, fileSystem.getInputStream(from)); + } + + // Copy the jars into the jar file. + for (String inputJar : inputJars) { + InputStream in = fileSystem.getInputStream(inputJar); + try { + combiner.addZip(inputJar, in); + InputStream inToClose = in; + in = null; + inToClose.close(); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + // Preserve original exception. + } + } + } + } + + // Close the output file. If something goes wrong here, delete the file. + combiner.close(); + combiner = null; + } finally { + // This part is only executed if an exception occurred. + if (combiner != null) { + try { + // We may end up calling close twice, but that's ok. + combiner.close(); + } catch (IOException e) { + // There's already an exception in progress - this won't add any + // additional information. + } + // Ignore return value - there's already an exception in progress. + fileSystem.delete(outputJar); + } + } + return 0; + } + + protected ZipEntryFilter createEntryFilter(boolean normalize, PathFilter allowedPaths) { + return new DefaultJarEntryFilter(normalize, allowedPaths); + } + + /** + * Collects the arguments for a command line flag until it finds a flag that + * starts with the terminatorPrefix. + * + * @param args + * @param startIndex the start index in the args to collect the flag arguments + * from + * @param flagArguments the collected flag arguments + * @param terminatorPrefix the terminator prefix to stop collecting of + * argument flags + * @return the index of the first argument that started with the + * terminatorPrefix + */ + private static int collectFlagArguments(List<String> args, int startIndex, + List<String> flagArguments, String terminatorPrefix) { + startIndex++; + while (startIndex < args.size()) { + String name = args.get(startIndex); + if (name.startsWith(terminatorPrefix)) { + return startIndex - 1; + } + flagArguments.add(name); + startIndex++; + } + return startIndex; + } + + /** + * Returns a single argument for a command line option. + * + * @throws IOException if no more arguments are available + */ + private static String getArgument(List<String> args, int i, String arg) throws IOException { + if (i + 1 < args.size()) { + return args.get(i + 1); + } + throw new IOException(arg + ": missing argument"); + } + + /** + * Processes the command line arguments. + * + * @throws IOException if one of the files containing options cannot be read + */ + protected void processCommandlineArgs(List<String> args) throws IOException { + List<String> manifestLines = new ArrayList<>(); + List<String> prefixes = new ArrayList<>(); + for (int i = 0; i < args.size(); i++) { + String arg = args.get(i); + if (arg.equals("--sources")) { + i = collectFlagArguments(args, i, inputJars, "--"); + } else if (arg.equals("--resources")) { + i = collectFlagArguments(args, i, resources, "--"); + } else if (arg.equals("--classpath_resources")) { + i = collectFlagArguments(args, i, classpathResources, "--"); + } else if (arg.equals("--deploy_manifest_lines")) { + i = collectFlagArguments(args, i, manifestLines, "--"); + } else if (arg.equals("--build_info_file")) { + buildInformationFiles.add(getArgument(args, i, arg)); + i++; + } else if (arg.equals("--extra_build_info")) { + buildInformations.add(getArgument(args, i, arg)); + i++; + } else if (arg.equals("--main_class")) { + mainClass = getArgument(args, i, arg); + i++; + } else if (arg.equals("--output")) { + outputJar = getArgument(args, i, arg); + i++; + } else if (arg.equals("--compression")) { + outputMode = OutputMode.FORCE_DEFLATE; + } else if (arg.equals("--dont_change_compression")) { + outputMode = OutputMode.DONT_CARE; + } else if (arg.equals("--normalize")) { + normalize = true; + } else if (arg.equals("--include_prefixes")) { + i = collectFlagArguments(args, i, prefixes, "--"); + } else if (arg.equals("--exclude_build_data")) { + includeBuildData = false; + } else if (arg.equals("--warn_duplicate_resources")) { + warnDuplicateFiles = true; + } else if (arg.equals("--java_launcher")) { + launcherBin = getArgument(args, i, arg); + i++; + } else { + throw new IOException("unknown option : '" + arg + "'"); + } + } + if (!manifestLines.isEmpty()) { + setExtraManifestContent(joinWithNewlines(manifestLines)); + } + if (!prefixes.isEmpty()) { + setPathPrefixes(prefixes); + } + } + + private String joinWithNewlines(Iterable<String> lines) { + StringBuilder result = new StringBuilder(); + Iterator<String> it = lines.iterator(); + if (it.hasNext()) { + result.append(it.next()); + } + while (it.hasNext()) { + result.append('\n'); + result.append(it.next()); + } + return result.toString(); + } + + private void setExtraManifestContent(String extraManifestContent) { + // The manifest content has to be terminated with a newline character + if (!extraManifestContent.endsWith("\n")) { + extraManifestContent = extraManifestContent + '\n'; + } + this.extraManifestContent = extraManifestContent; + } + + private void setPathPrefixes(List<String> prefixes) throws IOException { + if (prefixes.isEmpty()) { + throw new IOException( + "Empty set of path prefixes; cowardly refusing to emit an empty jar file"); + } + allowedPaths = new PrefixListPathFilter(prefixes); + } + + public static void main(String[] args) { + try { + SingleJar singlejar = new SingleJar(new JavaIoFileSystem()); + System.exit(singlejar.run(Arrays.asList(args))); + } catch (IOException e) { + System.err.println("SingleJar threw exception : " + e.getMessage()); + System.exit(1); + } + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java new file mode 100644 index 0000000000..d38c6d4493 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java @@ -0,0 +1,1643 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.singlejar; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.devtools.build.singlejar.ZipEntryFilter.CustomMergeStrategy; +import com.google.devtools.build.singlejar.ZipEntryFilter.StrategyCallback; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.CRC32; +import java.util.zip.DataFormatException; +import java.util.zip.Deflater; +import java.util.zip.Inflater; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.NotThreadSafe; + +/** + * An object that combines multiple ZIP files into a single file. It only + * supports a subset of the ZIP format, specifically: + * <ul> + * <li>It only supports STORE and DEFLATE storage methods.</li> + * <li>There may be no data before the first file or between files.</li> + * <li>It ignores any data after the last file.</li> + * </ul> + * + * <p>These restrictions are also present in the JDK implementations + * {@link java.util.jar.JarInputStream}, {@link java.util.zip.ZipInputStream}, + * though they are not documented there. + * + * <p>IMPORTANT NOTE: Callers must call {@link #finish()} or {@link #close()} + * at the end of processing to ensure that the output buffers are flushed and + * the ZIP file is complete. + * + * <p>This class performs only rudimentary data checking. If the input files + * are damaged, the output will likely also be damaged. + * + * <p>Also see: + * <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP format</a> + */ +@NotThreadSafe +public final class ZipCombiner implements AutoCloseable { + + /** + * A Date set to the 1/1/1980, 00:00:00, the minimum value that can be stored + * in a ZIP file. + */ + public static final Date DOS_EPOCH = new GregorianCalendar(1980, 0, 1, 0, 0, 0).getTime(); + + private static final int DEFAULT_CENTRAL_DIRECTORY_BLOCK_SIZE = 1048576; // 1 MB for each block + + // The following constants are ZIP-specific. + private static final int LOCAL_FILE_HEADER_MARKER = 0x04034b50; + private static final int DATA_DESCRIPTOR_MARKER = 0x08074b50; + private static final int CENTRAL_DIRECTORY_MARKER = 0x02014b50; + private static final int END_OF_CENTRAL_DIRECTORY_MARKER = 0x06054b50; + + private static final int FILE_HEADER_BUFFER_SIZE = 30; + + private static final int VERSION_TO_EXTRACT_OFFSET = 4; + private static final int GENERAL_PURPOSE_FLAGS_OFFSET = 6; + private static final int COMPRESSION_METHOD_OFFSET = 8; + private static final int MTIME_OFFSET = 10; + private static final int MDATE_OFFSET = 12; + private static final int CRC32_OFFSET = 14; + private static final int COMPRESSED_SIZE_OFFSET = 18; + private static final int UNCOMPRESSED_SIZE_OFFSET = 22; + private static final int FILENAME_LENGTH_OFFSET = 26; + private static final int EXTRA_LENGTH_OFFSET = 28; + + private static final int DIRECTORY_ENTRY_BUFFER_SIZE = 46; + + // Set if the size, compressed size and CRC are set to zero, and present in + // the data descriptor after the data. + private static final int SIZE_MASKED_FLAG = 1 << 3; + + private static final int STORED_METHOD = 0; + private static final int DEFLATE_METHOD = 8; + + private static final int VERSION_STORED = 10; // Version 1.0 + private static final int VERSION_DEFLATE = 20; // Version 2.0 + + private static final long MAXIMUM_DATA_SIZE = 0xffffffffL; + + // This class relies on the buffer to have sufficient space for a complete + // file name. 2^16 is the maximum number of bytes in a file name. + private static final int BUFFER_SIZE = 65536; + + /** An empty entry used to skip files that have already been copied (or skipped). */ + private static final FileEntry COPIED_FILE_ENTRY = new FileEntry(null, null, 0); + + /** An empty entry used to mark files that have already been renamed. */ + private static final FileEntry RENAMED_FILE_ENTRY = new FileEntry(null, null, 0); + + /** A zero length array of ExtraData. */ + public static final ExtraData[] NO_EXTRA_ENTRIES = new ExtraData[0]; + + /** + * Whether to compress or decompress entries. + */ + public enum OutputMode { + + /** + * Output entries using any method. + */ + DONT_CARE, + + /** + * Output all entries using DEFLATE method, except directory entries. It is + * always more efficient to store directory entries uncompressed. + */ + FORCE_DEFLATE, + + /** + * Output all entries using STORED method. + */ + FORCE_STORED; + } + + // A two-element enum for copyOrSkip type methods. + private static enum SkipMode { + + /** + * Copy the read data to the output stream. + */ + COPY, + + /** + * Do not write anything to the output stream. + */ + SKIP; + } + + /** + * Stores internal information about merges or skips. + */ + private static final class FileEntry { + + /** If null, the file should be skipped. Otherwise, it should be merged. */ + private final CustomMergeStrategy mergeStrategy; + private final ByteArrayOutputStream outputBuffer; + private final int dosTime; + + private FileEntry(CustomMergeStrategy mergeStrategy, ByteArrayOutputStream outputBuffer, + int dosTime) { + this.mergeStrategy = mergeStrategy; + this.outputBuffer = outputBuffer; + this.dosTime = dosTime; + } + } + + /** + * The directory entry info used for files whose extra directory entry info is not given + * explicitly. It uses {@code -1} for {@link DirectoryEntryInfo#withMadeByVersion(short)}, which + * indicates it will be set to the same version as "needed to extract." + * + * <p>The {@link DirectoryEntryInfo#withExternalFileAttribute(int)} value is set to {@code 0}, + * whose meaning depends on the value of {@code madeByVersion}, but is usually a reasonable + * default. + */ + public static final DirectoryEntryInfo DEFAULT_DIRECTORY_ENTRY_INFO = + new DirectoryEntryInfo((short) -1, 0); + + /** + * Contains information related to a zip entry that is stored in the central directory record. + * This does not contain all the information stored in the central directory record, only the + * information that can be customized and is not automatically calculated or detected. + */ + public static final class DirectoryEntryInfo { + private final short madeByVersion; + private final int externalFileAttribute; + + private DirectoryEntryInfo(short madeByVersion, int externalFileAttribute) { + this.madeByVersion = madeByVersion; + this.externalFileAttribute = externalFileAttribute; + } + + /** + * This will be written as "made by" version in the central directory. + * If -1 (default) then "made by" will be the same to version "needed to extract". + */ + public DirectoryEntryInfo withMadeByVersion(short madeByVersion) { + return new DirectoryEntryInfo(madeByVersion, externalFileAttribute); + } + + /** + * This will be written as external file attribute. The meaning of this depends upon the value + * set with {@link #withMadeByVersion(short)}. If that value indicates a Unix source, then this + * value has the file mode and permission bits in the upper two bytes (e.g. possibly + * {@code 0100644} for a regular file). + */ + public DirectoryEntryInfo withExternalFileAttribute(int externalFileAttribute) { + return new DirectoryEntryInfo(madeByVersion, externalFileAttribute); + } + } + + /** + * The central directory, which is grown as required; instead of using a single large buffer, we + * store a sequence of smaller buffers. With a single large buffer, whenever we grow the buffer by + * 2x, we end up requiring 3x the memory temporarily, which can lead to OOM problems even if there + * would still be enough memory. + * + * <p>The invariants for the fields are as follows: + * <ul> + * <li>All blocks must have the same size. + * <li>The list of blocks must contain all blocks, including the current block (even if empty). + * <li>The current block offset must apply to the last block in the list, which is + * simultaneously the current block. + * <li>The current block may only be {@code null} if the list is empty. + * </ul> + */ + private static final class CentralDirectory { + private final int blockSize; // We allow this to be overridden for testing. + private List<byte[]> blockList = new ArrayList<>(); + private byte[] currentBlock; + private int currentBlockOffset = 0; + private int size = 0; + + CentralDirectory(int centralDirectoryBlockSize) { + this.blockSize = centralDirectoryBlockSize; + } + + /** + * Appends the given data to the central directory and returns the start + * offset within the central directory to allow back-patching. + */ + int writeToCentralDirectory(byte[] b, int off, int len) { + checkArgument(len >= 0); + int offsetStarted = size; + while (len > 0) { + if (currentBlock == null + || currentBlockOffset >= currentBlock.length) { + currentBlock = new byte[blockSize]; + currentBlockOffset = 0; + blockList.add(currentBlock); + } + int maxCopy = Math.min(blockSize - currentBlockOffset, len); + System.arraycopy(b, off, currentBlock, currentBlockOffset, maxCopy); + off += maxCopy; + len -= maxCopy; + size += maxCopy; + currentBlockOffset += maxCopy; + } + return offsetStarted; + } + + /** Calls through to {@link #writeToCentralDirectory(byte[], int, int)}. */ + int writeToCentralDirectory(byte[] b) { + return writeToCentralDirectory(b, 0, b.length); + } + + /** + * Writes an unsigned int in little-endian byte order to the central directory at the + * given offset. Does not perform range checking. + */ + void setUnsignedInt(int offset, int value) { + blockList.get(cdIndex(offset + 0))[cdOffset(offset + 0)] = (byte) (value & 0xff); + blockList.get(cdIndex(offset + 1))[cdOffset(offset + 1)] = (byte) ((value >> 8) & 0xff); + blockList.get(cdIndex(offset + 2))[cdOffset(offset + 2)] = (byte) ((value >> 16) & 0xff); + blockList.get(cdIndex(offset + 3))[cdOffset(offset + 3)] = (byte) ((value >> 24) & 0xff); + } + + private int cdIndex(int offset) { + return offset / blockSize; + } + + private int cdOffset(int offset) { + return offset % blockSize; + } + + /** + * Writes the central directory to the given output stream and returns the size, i.e., the + * number of bytes written. + */ + int writeTo(OutputStream out) throws IOException { + for (int i = 0; i < blockList.size() - 1; i++) { + out.write(blockList.get(i)); + } + if (currentBlock != null) { + out.write(currentBlock, 0, currentBlockOffset); + } + return size; + } + } + + /** + * An output stream that counts how many bytes were written. + */ + private static final class ByteCountingOutputStream extends FilterOutputStream { + private long bytesWritten = 0L; + + ByteCountingOutputStream(OutputStream out) { + super(out); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + bytesWritten += len; + } + + @Override + public void write(int b) throws IOException { + out.write(b); + bytesWritten++; + } + } + + private final OutputMode mode; + private final ZipEntryFilter entryFilter; + + private final ByteCountingOutputStream out; + + // An input buffer to allow reading blocks of data. Keeping it here avoids + // another copy operation that would be required by the BufferedInputStream. + // The valid data is between bufferOffset and bufferOffset+bufferLength (exclusive). + private final byte[] buffer = new byte[BUFFER_SIZE]; + private int bufferOffset = 0; + private int bufferLength = 0; + + private String currentInputFile; + + // An intermediate buffer for the file header data. Keeping it here avoids + // creating a new buffer for every entry. + private final byte[] headerBuffer = new byte[FILE_HEADER_BUFFER_SIZE]; + + // An intermediate buffer for a central directory entry. Keeping it here + // avoids creating a new buffer for every entry. + private final byte[] directoryEntryBuffer = new byte[DIRECTORY_ENTRY_BUFFER_SIZE]; + + // The Inflater is a class member to avoid creating a new instance for every + // entry in the ZIP file. + private final Inflater inflater = new Inflater(true); + + // The contents of this buffer are never read. The Inflater is only used to + // determine the length of the compressed data, and the buffer is a throw- + // away buffer for the decompressed data. + private final byte[] inflaterBuffer = new byte[BUFFER_SIZE]; + + private final Map<String, FileEntry> fileNames = new HashMap<>(); + + private final CentralDirectory centralDirectory; + private int fileCount = 0; + + private boolean finished = false; + + // Package private for testing. + ZipCombiner(OutputMode mode, ZipEntryFilter entryFilter, OutputStream out, + int centralDirectoryBlockSize) { + this.mode = mode; + this.entryFilter = entryFilter; + this.out = new ByteCountingOutputStream(new BufferedOutputStream(out)); + this.centralDirectory = new CentralDirectory(centralDirectoryBlockSize); + } + + /** + * Creates a new instance with the given parameters. The {@code entryFilter} + * is called for every entry in the ZIP files and the combined ZIP file is + * written to {@code out}. The output mode determines whether entries must be + * written in compressed or decompressed form. Note that the result is + * invalid if an exception is thrown from any of the methods in this class, + * and before a call to {@link #close} or {@link #finish}. + */ + public ZipCombiner(OutputMode mode, ZipEntryFilter entryFilter, OutputStream out) { + this(mode, entryFilter, out, DEFAULT_CENTRAL_DIRECTORY_BLOCK_SIZE); + } + + /** + * Creates a new instance with the given parameters and the DONT_CARE mode. + */ + public ZipCombiner(ZipEntryFilter entryFilter, OutputStream out) { + this(OutputMode.DONT_CARE, entryFilter, out); + } + + /** + * Creates a new instance with the {@link CopyEntryFilter} as the filter and + * the given mode and output stream. + */ + public ZipCombiner(OutputMode mode, OutputStream out) { + this(mode, new CopyEntryFilter(), out); + } + + /** + * Creates a new instance with the {@link CopyEntryFilter} as the filter, the + * DONT_CARE mode and the given output stream. + */ + public ZipCombiner(OutputStream out) { + this(OutputMode.DONT_CARE, new CopyEntryFilter(), out); + } + + /** + * Returns whether the output zip already contains a file or directory with + * the given name. + */ + public boolean containsFile(String filename) { + return fileNames.containsKey(filename); + } + + /** + * Makes a write call to the output stream, and updates the current offset. + */ + private void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + } + + /** Calls through to {@link #write(byte[], int, int)}. */ + private void write(byte[] b) throws IOException { + write(b, 0, b.length); + } + + /** + * Reads at least one more byte into the internal buffer. This method must + * only be called when more data is necessary to correctly decode the ZIP + * format. + * + * <p>This method automatically compacts the existing data in the buffer by + * moving it to the beginning of the buffer. + * + * @throws EOFException if no more data is available from the input stream + * @throws IOException if the underlying stream throws one + */ + private void readMoreData(InputStream in) throws IOException { + if ((bufferLength > 0) && (bufferOffset > 0)) { + System.arraycopy(buffer, bufferOffset, buffer, 0, bufferLength); + } + if (bufferLength >= buffer.length) { + // The buffer size is specifically chosen to avoid this situation. + throw new AssertionError("Internal error: buffer overrun."); + } + bufferOffset = 0; + int bytesRead = in.read(buffer, bufferLength, buffer.length - bufferLength); + if (bytesRead <= 0) { + throw new EOFException(); + } + bufferLength += bytesRead; + } + + /** + * Reads data until the buffer is filled with at least {@code length} bytes. + * + * @throws IllegalArgumentException if not 0 <= length <= buffer.length + * @throws IOException if the underlying input stream throws one or the end + * of the input stream is reached before the required + * number of bytes is read + */ + private void readFully(InputStream in, int length) throws IOException { + checkArgument(length >= 0, "length too small: %s", length); + checkArgument(length <= buffer.length, "length too large: %s", length); + while (bufferLength < length) { + readMoreData(in); + } + } + + /** + * Reads an unsigned short in little-endian byte order from the buffer at the + * given offset. Does not perform range checking. + */ + private int getUnsignedShort(byte[] source, int offset) { + int a = source[offset + 0] & 0xff; + int b = source[offset + 1] & 0xff; + return (b << 8) | a; + } + + /** + * Reads an unsigned int in little-endian byte order from the buffer at the + * given offset. Does not perform range checking. + */ + private long getUnsignedInt(byte[] source, int offset) { + int a = source[offset + 0] & 0xff; + int b = source[offset + 1] & 0xff; + int c = source[offset + 2] & 0xff; + int d = source[offset + 3] & 0xff; + return ((d << 24) | (c << 16) | (b << 8) | a) & 0xffffffffL; + } + + /** + * Writes an unsigned short in little-endian byte order to the buffer at the + * given offset. Does not perform range checking. + */ + private void setUnsignedShort(byte[] target, int offset, short value) { + target[offset + 0] = (byte) (value & 0xff); + target[offset + 1] = (byte) ((value >> 8) & 0xff); + } + + /** + * Writes an unsigned int in little-endian byte order to the buffer at the + * given offset. Does not perform range checking. + */ + private void setUnsignedInt(byte[] target, int offset, int value) { + target[offset + 0] = (byte) (value & 0xff); + target[offset + 1] = (byte) ((value >> 8) & 0xff); + target[offset + 2] = (byte) ((value >> 16) & 0xff); + target[offset + 3] = (byte) ((value >> 24) & 0xff); + } + + /** + * Copies or skips {@code length} amount of bytes from the input stream to the + * output stream. If the internal buffer is not empty, those bytes are copied + * first. When the method returns, there may be more bytes remaining in the + * buffer. + * + * @throws IOException if the underlying stream throws one + */ + private void copyOrSkipData(InputStream in, long length, SkipMode skip) throws IOException { + checkArgument(length >= 0); + while (length > 0) { + if (bufferLength == 0) { + readMoreData(in); + } + int bytesToWrite = (length < bufferLength) ? (int) length : bufferLength; + if (skip == SkipMode.COPY) { + write(buffer, bufferOffset, bytesToWrite); + } + bufferOffset += bytesToWrite; + bufferLength -= bytesToWrite; + length -= bytesToWrite; + } + } + + /** + * Copies or skips {@code length} amount of bytes from the input stream to the + * output stream. If the internal buffer is not empty, those bytes are copied + * first. When the method returns, there may be more bytes remaining in the + * buffer. In addition to writing to the output stream, it also writes to the + * central directory. + * + * @throws IOException if the underlying stream throws one + */ + private void forkOrSkipData(InputStream in, long length, SkipMode skip) throws IOException { + checkArgument(length >= 0); + while (length > 0) { + if (bufferLength == 0) { + readMoreData(in); + } + int bytesToWrite = (length < bufferLength) ? (int) length : bufferLength; + if (skip == SkipMode.COPY) { + write(buffer, bufferOffset, bytesToWrite); + centralDirectory.writeToCentralDirectory(buffer, bufferOffset, bytesToWrite); + } + bufferOffset += bytesToWrite; + bufferLength -= bytesToWrite; + length -= bytesToWrite; + } + } + + /** + * A mutable integer reference value to allow returning two values from a + * method. + */ + private static class MutableInt { + + private int value; + + MutableInt(int initialValue) { + this.value = initialValue; + } + + public void setValue(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + } + + /** + * Uses the inflater to decompress some data into the given buffer. This + * method performs no error checking on the input parameters and also does + * not update the buffer parameters of the input buffer (such as bufferOffset + * and bufferLength). It's only here to avoid code duplication. + * + * <p>The Inflater may not be in the finished state when this method is + * called. + * + * <p>This method returns 0 if it read data and reached the end of the + * DEFLATE stream without producing output. In that case, {@link + * Inflater#finished} is guaranteed to return true. + * + * @throws IOException if the underlying stream throws an IOException or if + * illegal data is encountered + */ + private int inflateData(InputStream in, byte[] dest, int off, int len, MutableInt consumed) + throws IOException { + // Defend against Inflater.finished() returning true. + consumed.setValue(0); + int bytesProduced = 0; + int bytesConsumed = 0; + while ((bytesProduced == 0) && !inflater.finished()) { + inflater.setInput(buffer, bufferOffset + bytesConsumed, bufferLength - bytesConsumed); + int remainingBefore = inflater.getRemaining(); + try { + bytesProduced = inflater.inflate(dest, off, len); + } catch (DataFormatException e) { + throw new IOException("Invalid deflate stream in ZIP file.", e); + } + bytesConsumed += remainingBefore - inflater.getRemaining(); + consumed.setValue(bytesConsumed); + if (bytesProduced == 0) { + if (inflater.needsDictionary()) { + // The DEFLATE algorithm as used in the ZIP file format does not + // require an additional dictionary. + throw new AssertionError("Inflater unexpectedly requires a dictionary."); + } else if (inflater.needsInput()) { + readMoreData(in); + } else if (inflater.finished()) { + return 0; + } else { + // According to the Inflater specification, this cannot happen. + throw new AssertionError("Inflater unexpectedly produced no output."); + } + } + } + return bytesProduced; + } + + /** + * Copies or skips data from the input stream to the output stream. To + * determine the length of the data, the data is decompressed with the + * DEFLATE algorithm, which stores the length implicitly as part of the + * compressed data, using a combination of end markers and length indicators. + * + * @see <a href="http://www.ietf.org/rfc/rfc1951.txt">RFC 1951</a> + * + * @throws IOException if the underlying stream throws an IOException + */ + private long copyOrSkipDeflateData(InputStream in, SkipMode skip) throws IOException { + long bytesCopied = 0; + inflater.reset(); + MutableInt consumedBytes = new MutableInt(0); + while (!inflater.finished()) { + // Neither the uncompressed data nor the length of it is used. The + // decompression is only required to determine the correct length of the + // compressed data to copy. + inflateData(in, inflaterBuffer, 0, inflaterBuffer.length, consumedBytes); + int bytesRead = consumedBytes.getValue(); + if (skip == SkipMode.COPY) { + write(buffer, bufferOffset, bytesRead); + } + bufferOffset += bytesRead; + bufferLength -= bytesRead; + bytesCopied += bytesRead; + } + return bytesCopied; + } + + /** + * Returns a 32-bit integer containing a ZIP-compatible encoding of the given + * date. Only dates between 1980 and 2107 (inclusive) are supported. + * + * <p>The upper 16 bits contain the year, month, and day. The lower 16 bits + * contain the hour, minute, and second. The resolution of the second field + * is only 4 bits, which means that the only even second values can be + * stored - this method rounds down to the nearest even value. + * + * @throws IllegalArgumentException if the given date is outside the + * supported range + */ + // Only visible for testing. + static int dateToDosTime(Date date) { + Calendar calendar = new GregorianCalendar(); + calendar.setTime(date); + int year = calendar.get(Calendar.YEAR); + if (year < 1980) { + throw new IllegalArgumentException("date must be in or after 1980"); + } + // The ZIP format only provides 7 bits for the year. + if (year > 2107) { + throw new IllegalArgumentException("date must before 2107"); + } + int month = calendar.get(Calendar.MONTH) + 1; // Months from Calendar are zero-based. + int day = calendar.get(Calendar.DAY_OF_MONTH); + int hour = calendar.get(Calendar.HOUR_OF_DAY); + int minute = calendar.get(Calendar.MINUTE); + int second = calendar.get(Calendar.SECOND); + return ((year - 1980) << 25) | (month << 21) | (day << 16) + | (hour << 11) | (minute << 5) | (second >> 1); + } + + /** + * Fills the directory entry, using the information from the header buffer, + * and writes it to the central directory. It returns the offset into the + * central directory that can be used for patching the entry. Requires that + * the entire entry header is present in {@link #headerBuffer}. It also uses + * the {@link ByteCountingOutputStream#bytesWritten}, so it must be called + * just before the header is written to the output stream. + * + * @throws IOException if the current offset is too large for the ZIP format + */ + private int fillDirectoryEntryBuffer( + DirectoryEntryInfo directoryEntryInfo) throws IOException { + // central file header signature + setUnsignedInt(directoryEntryBuffer, 0, CENTRAL_DIRECTORY_MARKER); + short version = (short) getUnsignedShort(headerBuffer, VERSION_TO_EXTRACT_OFFSET); + short curMadeMyVersion = (directoryEntryInfo.madeByVersion == -1) + ? version : directoryEntryInfo.madeByVersion; + setUnsignedShort(directoryEntryBuffer, 4, curMadeMyVersion); // version made by + // version needed to extract + setUnsignedShort(directoryEntryBuffer, 6, version); + // general purpose bit flag + setUnsignedShort(directoryEntryBuffer, 8, + (short) getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET)); + // compression method + setUnsignedShort(directoryEntryBuffer, 10, + (short) getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET)); + // last mod file time, last mod file date + setUnsignedShort(directoryEntryBuffer, 12, + (short) getUnsignedShort(headerBuffer, MTIME_OFFSET)); + setUnsignedShort(directoryEntryBuffer, 14, + (short) getUnsignedShort(headerBuffer, MDATE_OFFSET)); + // crc-32 + setUnsignedInt(directoryEntryBuffer, 16, (int) getUnsignedInt(headerBuffer, CRC32_OFFSET)); + // compressed size + setUnsignedInt(directoryEntryBuffer, 20, + (int) getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET)); + // uncompressed size + setUnsignedInt(directoryEntryBuffer, 24, + (int) getUnsignedInt(headerBuffer, UNCOMPRESSED_SIZE_OFFSET)); + // file name length + setUnsignedShort(directoryEntryBuffer, 28, + (short) getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET)); + // extra field length + setUnsignedShort(directoryEntryBuffer, 30, + (short) getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET)); + setUnsignedShort(directoryEntryBuffer, 32, (short) 0); // file comment length + setUnsignedShort(directoryEntryBuffer, 34, (short) 0); // disk number start + setUnsignedShort(directoryEntryBuffer, 36, (short) 0); // internal file attributes + setUnsignedInt(directoryEntryBuffer, 38, directoryEntryInfo.externalFileAttribute); + if (out.bytesWritten >= MAXIMUM_DATA_SIZE) { + throw new IOException("Unable to handle files bigger than 2^32 bytes."); + } + // relative offset of local header + setUnsignedInt(directoryEntryBuffer, 42, (int) out.bytesWritten); + fileCount++; + return centralDirectory.writeToCentralDirectory(directoryEntryBuffer); + } + + /** + * Fix the directory entry with the correct crc32, compressed size, and + * uncompressed size. + */ + private void fixDirectoryEntry(int offset, long crc32, long compressedSize, + long uncompressedSize) { + // The constants from the top don't apply here, because this is the central directory entry. + centralDirectory.setUnsignedInt(offset + 16, (int) crc32); // crc-32 + centralDirectory.setUnsignedInt(offset + 20, (int) compressedSize); // compressed size + centralDirectory.setUnsignedInt(offset + 24, (int) uncompressedSize); // uncompressed size + } + + /** + * (Un)Compresses and copies the current ZIP file entry. Requires that the + * entire entry header is present in {@link #headerBuffer}. It currently + * drops the extra data in the process. + * + * @throws IOException if the underlying stream throws an IOException + */ + private void modifyAndCopyEntry(String filename, InputStream in, int dosTime) + throws IOException { + final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); + final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET); + final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET); + final int extraFieldLength = getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET); + // TODO(bazel-team): Read and copy the extra data if present. + + forkOrSkipData(in, fileNameLength, SkipMode.SKIP); + forkOrSkipData(in, extraFieldLength, SkipMode.SKIP); + if (method == STORED_METHOD) { + long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET); + copyStreamToEntry(filename, new FixedLengthInputStream(in, compressedSize), dosTime, + NO_EXTRA_ENTRIES, true, DEFAULT_DIRECTORY_ENTRY_INFO); + } else if (method == DEFLATE_METHOD) { + inflater.reset(); + copyStreamToEntry(filename, new DeflateInputStream(in), dosTime, NO_EXTRA_ENTRIES, false, + DEFAULT_DIRECTORY_ENTRY_INFO); + if ((flags & SIZE_MASKED_FLAG) != 0) { + copyOrSkipData(in, 16, SkipMode.SKIP); + } + } else { + throw new AssertionError("This should have been checked in validateHeader()."); + } + } + + /** + * Copies or skips the current ZIP file entry. Requires that the entire entry + * header is present in {@link #headerBuffer}. It uses the current mode to + * decide whether to compress or decompress the entry. + * + * @throws IOException if the underlying stream throws an IOException + */ + private void copyOrSkipEntry(String filename, InputStream in, SkipMode skip, Date date, + DirectoryEntryInfo directoryEntryInfo) throws IOException { + copyOrSkipEntry(filename, in, skip, date, directoryEntryInfo, false); + } + + /** + * Renames and otherwise copies the current ZIP file entry. Requires that the entire + * entry header is present in {@link #headerBuffer}. It uses the current mode to + * decide whether to compress or decompress the entry. + * + * @throws IOException if the underlying stream throws an IOException + */ + private void renameEntry(String filename, InputStream in, Date date, + DirectoryEntryInfo directoryEntryInfo) throws IOException { + copyOrSkipEntry(filename, in, SkipMode.COPY, date, directoryEntryInfo, true); + } + + /** + * Copies or skips the current ZIP file entry. Requires that the entire entry + * header is present in {@link #headerBuffer}. It uses the current mode to + * decide whether to compress or decompress the entry. + * + * @throws IOException if the underlying stream throws an IOException + */ + private void copyOrSkipEntry(String filename, InputStream in, SkipMode skip, Date date, + DirectoryEntryInfo directoryEntryInfo, boolean rename) throws IOException { + final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); + + // We can cast here, because the result is only treated as a bitmask. + int dosTime = date == null ? (int) getUnsignedInt(headerBuffer, MTIME_OFFSET) + : dateToDosTime(date); + if (skip == SkipMode.COPY) { + if ((mode == OutputMode.FORCE_DEFLATE) && (method == STORED_METHOD) + && !filename.endsWith("/")) { + modifyAndCopyEntry(filename, in, dosTime); + return; + } else if ((mode == OutputMode.FORCE_STORED) && (method == DEFLATE_METHOD)) { + modifyAndCopyEntry(filename, in, dosTime); + return; + } + } + + int directoryOffset = copyOrSkipEntryHeader(filename, in, date, directoryEntryInfo, + skip, rename); + + copyOrSkipEntryData(filename, in, skip, directoryOffset); + } + + /** + * Copies or skips the header of an entry, including filename and extra data. + * Requires that the entire entry header is present in {@link #headerBuffer}. + * + * @returns the enrty offset in the central directory + * @throws IOException if the underlying stream throws an IOException + */ + private int copyOrSkipEntryHeader(String filename, InputStream in, Date date, + DirectoryEntryInfo directoryEntryInfo, SkipMode skip, boolean rename) + throws IOException { + final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET); + final int extraFieldLength = getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET); + + byte[] fileNameAsBytes = null; + if (rename) { + // If the entry is renamed, we patch the filename length in the buffer + // before it's copied, and before writing to the central directory. + fileNameAsBytes = filename.getBytes(UTF_8); + checkArgument(fileNameAsBytes.length <= 65535, + "File name too long: %s bytes (max. 65535)", fileNameAsBytes.length); + setUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET, (short) fileNameAsBytes.length); + } + + int directoryOffset = 0; + if (skip == SkipMode.COPY) { + if (date != null) { + int dosTime = dateToDosTime(date); + setUnsignedShort(headerBuffer, MTIME_OFFSET, (short) dosTime); // lower 16 bits + setUnsignedShort(headerBuffer, MDATE_OFFSET, (short) (dosTime >> 16)); // upper 16 bits + } + // Call this before writing the data out, so that we get the correct offset. + directoryOffset = fillDirectoryEntryBuffer(directoryEntryInfo); + write(headerBuffer, 0, FILE_HEADER_BUFFER_SIZE); + } + if (!rename) { + forkOrSkipData(in, fileNameLength, skip); + } else { + forkOrSkipData(in, fileNameLength, SkipMode.SKIP); + write(fileNameAsBytes); + centralDirectory.writeToCentralDirectory(fileNameAsBytes); + } + forkOrSkipData(in, extraFieldLength, skip); + return directoryOffset; + } + + /** + * Copy or skip the data of an entry. Requires that the + * entire entry header is present in {@link #headerBuffer}. + * + * @throws IOException if the underlying stream throws an IOException + */ + private void copyOrSkipEntryData(String filename, InputStream in, SkipMode skip, + int directoryOffset) throws IOException { + final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET); + final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); + if ((flags & SIZE_MASKED_FLAG) != 0) { + // The compressed data size is unknown. + if (method != DEFLATE_METHOD) { + throw new AssertionError("This should have been checked in validateHeader()."); + } + copyOrSkipDeflateData(in, skip); + // The flags indicate that a data descriptor must follow the data. + readFully(in, 16); + if (getUnsignedInt(buffer, bufferOffset) != DATA_DESCRIPTOR_MARKER) { + throw new IOException("Missing data descriptor for " + filename + " in " + currentInputFile + + "."); + } + long crc32 = getUnsignedInt(buffer, bufferOffset + 4); + long compressedSize = getUnsignedInt(buffer, bufferOffset + 8); + long uncompressedSize = getUnsignedInt(buffer, bufferOffset + 12); + if (skip == SkipMode.COPY) { + fixDirectoryEntry(directoryOffset, crc32, compressedSize, uncompressedSize); + } + copyOrSkipData(in, 16, skip); + } else { + // The size value is present in the header, so just copy that amount. + long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET); + copyOrSkipData(in, compressedSize, skip); + } + } + + /** + * An input stream that reads a fixed number of bytes from the given input + * stream before it returns end-of-input. It uses the local buffer, so it + * can't be static. + */ + private class FixedLengthInputStream extends InputStream { + + private final InputStream in; + private long remainingBytes; + private final byte[] singleByteBuffer = new byte[1]; + + FixedLengthInputStream(InputStream in, long remainingBytes) { + this.in = in; + this.remainingBytes = remainingBytes; + } + + @Override + public int read() throws IOException { + int bytesRead = read(singleByteBuffer, 0, 1); + return (bytesRead == -1) ? -1 : singleByteBuffer[0]; + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + checkArgument(len >= 0); + checkArgument(off >= 0); + checkArgument(off + len <= b.length); + if (remainingBytes == 0) { + return -1; + } + if (bufferLength == 0) { + readMoreData(in); + } + int bytesToCopy = len; + if (remainingBytes < bytesToCopy) { + bytesToCopy = (int) remainingBytes; + } + if (bufferLength < bytesToCopy) { + bytesToCopy = bufferLength; + } + System.arraycopy(buffer, bufferOffset, b, off, bytesToCopy); + bufferOffset += bytesToCopy; + bufferLength -= bytesToCopy; + remainingBytes -= bytesToCopy; + return bytesToCopy; + } + } + + /** + * An input stream that reads from a given input stream, decoding that data + * according to the DEFLATE algorithm. The DEFLATE data stream implicitly + * contains its own end-of-input marker. It uses the local buffer, so it + * can't be static. + */ + private class DeflateInputStream extends InputStream { + + private final InputStream in; + private final byte[] singleByteBuffer = new byte[1]; + private final MutableInt consumedBytes = new MutableInt(0); + + DeflateInputStream(InputStream in) { + this.in = in; + } + + @Override + public int read() throws IOException { + int bytesRead = read(singleByteBuffer, 0, 1); + // Do an unsigned cast on the byte from the buffer if it exists. + return (bytesRead == -1) ? -1 : (singleByteBuffer[0] & 0xff); + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + if (inflater.finished()) { + return -1; + } + int length = inflateData(in, b, off, len, consumedBytes); + int bytesRead = consumedBytes.getValue(); + bufferOffset += bytesRead; + bufferLength -= bytesRead; + return length == 0 ? -1 : length; + } + } + + /** + * Handles a custom merge operation with the given strategy. This method + * creates an appropriate input stream and hands it to the strategy for + * processing. Requires that the entire entry header is present in {@link + * #headerBuffer}. + * + * @throws IOException if one of the underlying stream throws an IOException, + * if the ZIP entry data is inconsistent, or if the + * implementation cannot handle the compression method + * given in the ZIP entry + */ + private void handleCustomMerge(final InputStream in, CustomMergeStrategy mergeStrategy, + ByteArrayOutputStream outputBuffer) throws IOException { + final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET); + final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); + final long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET); + + final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET); + final int extraFieldLength = getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET); + + copyOrSkipData(in, fileNameLength, SkipMode.SKIP); + copyOrSkipData(in, extraFieldLength, SkipMode.SKIP); + if (method == STORED_METHOD) { + mergeStrategy.merge(new FixedLengthInputStream(in, compressedSize), outputBuffer); + } else if (method == DEFLATE_METHOD) { + inflater.reset(); + // TODO(bazel-team): Defend against the mergeStrategy not reading the complete input. + mergeStrategy.merge(new DeflateInputStream(in), outputBuffer); + if ((flags & SIZE_MASKED_FLAG) != 0) { + copyOrSkipData(in, 16, SkipMode.SKIP); + } + } else { + throw new AssertionError("This should have been checked in validateHeader()."); + } + } + + /** + * Implementation of the strategy callback. + */ + private class TheStrategyCallback implements StrategyCallback { + + private String filename; + private final InputStream in; + + // Use an atomic boolean to make sure that only a single call goes + // through, even if there are multiple concurrent calls. Paranoid + // defensive programming. + private final AtomicBoolean callDone = new AtomicBoolean(); + + TheStrategyCallback(String filename, InputStream in) { + this.filename = filename; + this.in = in; + } + + // Verify that this is the first call and throw an exception if not. + private void checkCall() { + checkState(callDone.compareAndSet(false, true), "The callback was already called once."); + } + + @Override + public void copy(Date date) throws IOException { + checkCall(); + if (!containsFile(filename)) { + fileNames.put(filename, COPIED_FILE_ENTRY); + copyOrSkipEntry(filename, in, SkipMode.COPY, date, DEFAULT_DIRECTORY_ENTRY_INFO); + } else { // can't copy, name already used for renamed entry + copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO); + } + } + + @Override + public void rename(String newName, Date date) throws IOException { + checkCall(); + if (!containsFile(newName)) { + fileNames.put(newName, RENAMED_FILE_ENTRY); + renameEntry(newName, in, date, DEFAULT_DIRECTORY_ENTRY_INFO); + } else { + copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO); + } + filename = newName; + } + + @Override + public void skip() throws IOException { + checkCall(); + if (!containsFile(filename)) {// don't overwrite possible RENAMED_FILE_ENTRY value + fileNames.put(filename, COPIED_FILE_ENTRY); + } + copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO); + } + + @Override + public void customMerge(Date date, CustomMergeStrategy strategy) throws IOException { + checkCall(); + ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream(); + fileNames.put(filename, new FileEntry(strategy, outputBuffer, dateToDosTime(date))); + handleCustomMerge(in, strategy, outputBuffer); + } + } + + /** + * Validates that the current entry obeys all the restrictions of this implementation. + * + * @throws IOException if the current entry doesn't obey the restrictions + */ + private void validateHeader() throws IOException { + // We only handle DEFLATE and STORED, like java.util.zip. + final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); + if ((method != DEFLATE_METHOD) && (method != STORED_METHOD)) { + throw new IOException("Unable to handle compression methods other than DEFLATE!"); + } + + // If the method is STORED, then the size must be available in the header. + final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET); + if ((method == STORED_METHOD) && ((flags & SIZE_MASKED_FLAG) != 0)) { + throw new IOException("If the method is STORED, then the size must be available in the" + + " header!"); + } + + // If the method is STORED, the compressed and uncompressed sizes must be equal. + final long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET); + final long uncompressedSize = getUnsignedInt(headerBuffer, UNCOMPRESSED_SIZE_OFFSET); + if ((method == STORED_METHOD) && (compressedSize != uncompressedSize)) { + throw new IOException("Compressed and uncompressed sizes for STORED entry differ!"); + } + + // The compressed or uncompressed size being set to 0xffffffff is a strong indicator that the + // ZIP file is in ZIP64 mode, which supports files larger than 2^32. + // TODO(bazel-team): Support the ZIP64 extension. + if ((compressedSize == MAXIMUM_DATA_SIZE) || (uncompressedSize == MAXIMUM_DATA_SIZE)) { + throw new IOException("Unable to handle ZIP64 compressed files."); + } + } + + /** + * Reads a file entry from the input stream, calls the entryFilter to + * determine what to do with the entry, and performs the requested operation. + * Returns true if the input stream contained another entry. + * + * @throws IOException if one of the underlying stream throws an IOException, + * if the ZIP contains unsupported, inconsistent or + * incomplete data or if the filter throws an IOException + */ + private boolean handleNextEntry(final InputStream in) throws IOException { + // Just try to read the complete header and fail if it didn't work. + try { + readFully(in, FILE_HEADER_BUFFER_SIZE); + } catch (EOFException e) { + return false; + } + + System.arraycopy(buffer, bufferOffset, headerBuffer, 0, FILE_HEADER_BUFFER_SIZE); + bufferOffset += FILE_HEADER_BUFFER_SIZE; + bufferLength -= FILE_HEADER_BUFFER_SIZE; + if (getUnsignedInt(headerBuffer, 0) != LOCAL_FILE_HEADER_MARKER) { + return false; + } + validateHeader(); + + final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET); + readFully(in, fileNameLength); + // TODO(bazel-team): If I read the spec correctly, this should be UTF-8 rather than ISO-8859-1. + final String filename = new String(buffer, bufferOffset, fileNameLength, ISO_8859_1); + + FileEntry handler = fileNames.get(filename); + // The handler is null if this is the first time we see an entry with this filename, + // or if all previous entries with this name were renamed by the filter (and we can + // pretend we didn't encounter the name yet). + // If the handler is RENAMED_FILE_ENTRY, a previous entry was renamed as filename, + // in which case the filter should now be invoked for this name for the first time, + // giving the filter a chance to choose an unique name. + if (handler == null || handler == RENAMED_FILE_ENTRY) { + TheStrategyCallback callback = new TheStrategyCallback(filename, in); + entryFilter.accept(filename, callback); + if (fileNames.get(callback.filename) == null && fileNames.get(filename) == null) { + throw new IllegalStateException(); + } + } else if (handler.mergeStrategy == null) { + copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO); + } else { + handleCustomMerge(in, handler.mergeStrategy, handler.outputBuffer); + } + return true; + } + + /** + * Clears the internal buffer. + */ + private void clearBuffer() { + bufferOffset = 0; + bufferLength = 0; + } + + /** + * Copies another ZIP file into the output. If multiple entries with the same + * name are present, the first such entry is copied, but the others are + * ignored. This is also true for multiple invocations of this method. The + * {@code inputName} parameter is used to provide better error messages in the + * case of a failure to decode the ZIP file. + * + * @throws IOException if one of the underlying stream throws an IOException, + * if the ZIP contains unsupported, inconsistent or + * incomplete data or if the filter throws an IOException + */ + public void addZip(String inputName, InputStream in) throws IOException { + if (finished) { + throw new IllegalStateException(); + } + if (in == null) { + throw new NullPointerException(); + } + clearBuffer(); + currentInputFile = inputName; + while (handleNextEntry(in)) {/*handleNextEntry has side-effect.*/} + } + + public void addZip(InputStream in) throws IOException { + addZip(null, in); + } + + private void copyStreamToEntry(String filename, InputStream in, int dosTime, + ExtraData[] extraDataEntries, boolean compress, DirectoryEntryInfo directoryEntryInfo) + throws IOException { + fileNames.put(filename, COPIED_FILE_ENTRY); + + byte[] fileNameAsBytes = filename.getBytes(UTF_8); + checkArgument(fileNameAsBytes.length <= 65535, + "File name too long: %s bytes (max. 65535)", fileNameAsBytes.length); + + // Note: This method can be called with an input stream that uses the buffer field of this + // class. We use a local buffer here to avoid conflicts. + byte[] localBuffer = new byte[4096]; + + byte[] uncompressedData = null; + if (!compress) { + ByteArrayOutputStream temp = new ByteArrayOutputStream(); + int bytesRead; + while ((bytesRead = in.read(localBuffer)) != -1) { + temp.write(localBuffer, 0, bytesRead); + } + uncompressedData = temp.toByteArray(); + } + byte[] extraData = null; + if (extraDataEntries.length != 0) { + int totalLength = 0; + for (ExtraData extra : extraDataEntries) { + int length = extra.getData().length; + if (totalLength > 0xffff - 4 - length) { + throw new IOException("Total length of extra data too big."); + } + totalLength += length + 4; + } + extraData = new byte[totalLength]; + int position = 0; + for (ExtraData extra : extraDataEntries) { + byte[] data = extra.getData(); + setUnsignedShort(extraData, position + 0, extra.getId()); + setUnsignedShort(extraData, position + 2, (short) data.length); + System.arraycopy(data, 0, extraData, position + 4, data.length); + position += data.length + 4; + } + } + + // write header + Arrays.fill(headerBuffer, (byte) 0); + setUnsignedInt(headerBuffer, 0, LOCAL_FILE_HEADER_MARKER); // file header signature + if (compress) { + setUnsignedShort(headerBuffer, 4, (short) VERSION_DEFLATE); // version to extract + setUnsignedShort(headerBuffer, 6, (short) SIZE_MASKED_FLAG); // general purpose bit flag + setUnsignedShort(headerBuffer, 8, (short) DEFLATE_METHOD); // compression method + } else { + setUnsignedShort(headerBuffer, 4, (short) VERSION_STORED); // version to extract + setUnsignedShort(headerBuffer, 6, (short) 0); // general purpose bit flag + setUnsignedShort(headerBuffer, 8, (short) STORED_METHOD); // compression method + } + setUnsignedShort(headerBuffer, 10, (short) dosTime); // mtime + setUnsignedShort(headerBuffer, 12, (short) (dosTime >> 16)); // mdate + if (uncompressedData != null) { + CRC32 crc = new CRC32(); + crc.update(uncompressedData); + setUnsignedInt(headerBuffer, 14, (int) crc.getValue()); // crc32 + setUnsignedInt(headerBuffer, 18, uncompressedData.length); // compressed size + setUnsignedInt(headerBuffer, 22, uncompressedData.length); // uncompressed size + } else { + setUnsignedInt(headerBuffer, 14, 0); // crc32 + setUnsignedInt(headerBuffer, 18, 0); // compressed size + setUnsignedInt(headerBuffer, 22, 0); // uncompressed size + } + setUnsignedShort(headerBuffer, 26, (short) fileNameAsBytes.length); // file name length + if (extraData != null) { + setUnsignedShort(headerBuffer, 28, (short) extraData.length); // extra field length + } else { + setUnsignedShort(headerBuffer, 28, (short) 0); // extra field length + } + + // This call works for both compressed or uncompressed entries. + int directoryOffset = fillDirectoryEntryBuffer(directoryEntryInfo); + write(headerBuffer); + write(fileNameAsBytes); + centralDirectory.writeToCentralDirectory(fileNameAsBytes); + if (extraData != null) { + write(extraData); + centralDirectory.writeToCentralDirectory(extraData); + } + + // write data + if (uncompressedData != null) { + write(uncompressedData); + } else { + try (DeflaterOutputStream deflaterStream = new DeflaterOutputStream()) { + int bytesRead; + while ((bytesRead = in.read(localBuffer)) != -1) { + deflaterStream.write(localBuffer, 0, bytesRead); + } + deflaterStream.finish(); + + // write data descriptor + Arrays.fill(headerBuffer, (byte) 0); + setUnsignedInt(headerBuffer, 0, DATA_DESCRIPTOR_MARKER); + setUnsignedInt(headerBuffer, 4, deflaterStream.getCRC()); // crc32 + setUnsignedInt(headerBuffer, 8, deflaterStream.getCompressedSize()); // compressed size + setUnsignedInt(headerBuffer, 12, deflaterStream.getUncompressedSize()); // uncompressed size + write(headerBuffer, 0, 16); + fixDirectoryEntry(directoryOffset, deflaterStream.getCRC(), + deflaterStream.getCompressedSize(), deflaterStream.getUncompressedSize()); + } + } + } + + /** + * Adds a new entry into the output, by reading the input stream until it + * returns end of stream. Equivalent to + * {@link #addFile(String, Date, InputStream, DirectoryEntryInfo)}, but uses + * {@link #DEFAULT_DIRECTORY_ENTRY_INFO} for the file's directory entry. + */ + public void addFile(String filename, Date date, InputStream in) throws IOException { + addFile(filename, date, in, DEFAULT_DIRECTORY_ENTRY_INFO); + } + + /** + * Adds a new entry into the output, by reading the input stream until it + * returns end of stream. This method does not call {@link + * ZipEntryFilter#accept}. + * + * @throws IOException if one of the underlying streams throws an IOException + * or if the input stream returns more data than + * supported by the ZIP format + * @throws IllegalStateException if an entry with the given name already + * exists + * @throws IllegalArgumentException if the given file name is longer than + * supported by the ZIP format + */ + public void addFile(String filename, Date date, InputStream in, + DirectoryEntryInfo directoryEntryInfo) throws IOException { + checkNotFinished(); + if (in == null) { + throw new NullPointerException(); + } + if (filename == null) { + throw new NullPointerException(); + } + checkState(!fileNames.containsKey(filename), + "jar already contains a file named %s", filename); + int dosTime = dateToDosTime(date != null ? date : new Date()); + copyStreamToEntry(filename, in, dosTime, NO_EXTRA_ENTRIES, + mode != OutputMode.FORCE_STORED, // Always compress if we're allowed to. + directoryEntryInfo); + } + + /** + * Adds a new directory entry into the output. This method does not call + * {@link ZipEntryFilter#accept}. Uses {@link #DEFAULT_DIRECTORY_ENTRY_INFO} for the added + * directory entry. + * + * @throws IOException if one of the underlying streams throws an IOException + * @throws IllegalStateException if an entry with the given name already + * exists + * @throws IllegalArgumentException if the given file name is longer than + * supported by the ZIP format + */ + public void addDirectory(String filename, Date date, ExtraData[] extraDataEntries) + throws IOException { + checkNotFinished(); + checkArgument(filename.endsWith("/")); // Can also throw NPE. + checkState(!fileNames.containsKey(filename), + "jar already contains a directory named %s", filename); + int dosTime = dateToDosTime(date != null ? date : new Date()); + copyStreamToEntry(filename, new ByteArrayInputStream(new byte[0]), dosTime, extraDataEntries, + false, // Never compress directory entries. + DEFAULT_DIRECTORY_ENTRY_INFO); + } + + /** + * Adds a new directory entry into the output. This method does not call + * {@link ZipEntryFilter#accept}. + * + * @throws IOException if one of the underlying streams throws an IOException + * @throws IllegalStateException if an entry with the given name already + * exists + * @throws IllegalArgumentException if the given file name is longer than + * supported by the ZIP format + */ + public void addDirectory(String filename, Date date) + throws IOException { + addDirectory(filename, date, NO_EXTRA_ENTRIES); + } + + /** + * A deflater output stream that also counts uncompressed and compressed + * numbers of bytes and computes the CRC so that the data descriptor marker + * is written correctly. + * + * <p>Not static, so it can access the write() methods. + */ + private class DeflaterOutputStream extends OutputStream { + + private final Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, true); + private final CRC32 crc = new CRC32(); + private final byte[] outputBuffer = new byte[4096]; + private long uncompressedBytes = 0; + private long compressedBytes = 0; + + @Override + public void write(int b) throws IOException { + byte[] buf = new byte[] { (byte) (b & 0xff) }; + write(buf, 0, buf.length); + } + + @Override + public void write(byte b[], int off, int len) throws IOException { + checkNotFinished(); + uncompressedBytes += len; + crc.update(b, off, len); + deflater.setInput(b, off, len); + while (!deflater.needsInput()) { + deflate(); + } + } + + @Override + public void close() throws IOException { + super.close(); + deflater.end(); + } + + /** + * Writes out the remaining buffered data without closing the output + * stream. + */ + public void finish() throws IOException { + checkNotFinished(); + deflater.finish(); + while (!deflater.finished()) { + deflate(); + } + if ((compressedBytes >= MAXIMUM_DATA_SIZE) || (uncompressedBytes >= MAXIMUM_DATA_SIZE)) { + throw new IOException("Too much data for ZIP entry."); + } + } + + private void deflate() throws IOException { + int length = deflater.deflate(outputBuffer); + ZipCombiner.this.write(outputBuffer, 0, length); + compressedBytes += length; + } + + public int getCRC() { + return (int) crc.getValue(); + } + + public int getCompressedSize() { + return (int) compressedBytes; + } + + public int getUncompressedSize() { + return (int) uncompressedBytes; + } + + private void checkNotFinished() { + if (deflater.finished()) { + throw new IllegalStateException(); + } + } + } + + /** + * Writes any remaining output data to the output stream and also creates the + * merged entries by calling the {@link CustomMergeStrategy} implementations + * given back from the ZIP entry filter. + * + * @throws IOException if the output stream or the filter throws an + * IOException + * @throws IllegalStateException if this method was already called earlier + */ + public void finish() throws IOException { + checkNotFinished(); + finished = true; + for (Map.Entry<String, FileEntry> entry : fileNames.entrySet()) { + String filename = entry.getKey(); + CustomMergeStrategy mergeStrategy = entry.getValue().mergeStrategy; + ByteArrayOutputStream outputBuffer = entry.getValue().outputBuffer; + int dosTime = entry.getValue().dosTime; + if (mergeStrategy == null) { + // Do nothing. + } else { + mergeStrategy.finish(outputBuffer); + copyStreamToEntry(filename, new ByteArrayInputStream(outputBuffer.toByteArray()), dosTime, + NO_EXTRA_ENTRIES, true, DEFAULT_DIRECTORY_ENTRY_INFO); + } + } + + // Write central directory. + if (out.bytesWritten >= MAXIMUM_DATA_SIZE) { + throw new IOException("Unable to handle files bigger than 2^32 bytes."); + } + int startOfCentralDirectory = (int) out.bytesWritten; + int centralDirectorySize = centralDirectory.writeTo(out); + + // end of central directory signature + setUnsignedInt(directoryEntryBuffer, 0, END_OF_CENTRAL_DIRECTORY_MARKER); + // number of this disk + setUnsignedShort(directoryEntryBuffer, 4, (short) 0); + // number of the disk with the start of the central directory + setUnsignedShort(directoryEntryBuffer, 6, (short) 0); + // total number of entries in the central directory on this disk + setUnsignedShort(directoryEntryBuffer, 8, (short) fileCount); + // total number of entries in the central directory + setUnsignedShort(directoryEntryBuffer, 10, (short) fileCount); + // size of the central directory + setUnsignedInt(directoryEntryBuffer, 12, centralDirectorySize); + // offset of start of central directory with respect to the starting disk number + setUnsignedInt(directoryEntryBuffer, 16, startOfCentralDirectory); + // .ZIP file comment length + setUnsignedShort(directoryEntryBuffer, 20, (short) 0); + write(directoryEntryBuffer, 0, 22); + + out.flush(); + } + + private void checkNotFinished() { + if (finished) { + throw new IllegalStateException(); + } + } + + /** + * Writes any remaining output data to the output stream and closes it. + * + * @throws IOException if the output stream or the filter throws an + * IOException + */ + @Override + public void close() throws IOException { + if (!finished) { + finish(); + } + out.close(); + } + + /** + * Turns this JAR file into an executable JAR by prepending an executable. + * JAR files are placed at the end of a file, and executables are placed + * at the beginning, so a file can be both, if desired. + * + * @param launcherIn The InputStream, from which the launcher is read. + * @throws NullPointerException if launcherIn is null + * @throws IOException if reading from launcherIn or writing to the output + * stream throws an IOException. + */ + public void prependExecutable(InputStream launcherIn) throws IOException { + if (launcherIn == null) { + throw new NullPointerException("No launcher specified"); + } + byte[] buf = new byte[BUFFER_SIZE]; + int bytesRead; + while ((bytesRead = launcherIn.read(buf)) > 0) { + out.write(buf, 0, bytesRead); + } + } + + /** + * Ensures the truth of an expression involving one or more parameters to the calling method. + */ + private static void checkArgument(boolean expression, + @Nullable String errorMessageTemplate, + @Nullable Object... errorMessageArgs) { + if (!expression) { + throw new IllegalArgumentException(String.format(errorMessageTemplate, errorMessageArgs)); + } + } + + /** + * Ensures the truth of an expression involving one or more parameters to the calling method. + */ + private static void checkArgument(boolean expression) { + if (!expression) { + throw new IllegalArgumentException(); + } + } + + /** + * Ensures the truth of an expression involving state. + */ + private static void checkState(boolean expression, + @Nullable String errorMessageTemplate, + @Nullable Object... errorMessageArgs) { + if (!expression) { + throw new IllegalStateException(String.format(errorMessageTemplate, errorMessageArgs)); + } + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipEntryFilter.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipEntryFilter.java new file mode 100644 index 0000000000..ab5a24af36 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipEntryFilter.java @@ -0,0 +1,119 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.singlejar; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Date; + +/** + * A custom filter for entries when combining multiple ZIP files (or even just + * copying a single ZIP file). + * + * <p>Implementations of this interface must be thread-safe. The {@link + * #accept} method may be called concurrently by multiple threads. + */ +public interface ZipEntryFilter { + + /** + * Strategy for a custom merge operation. The current file and all additional + * file are passed to the strategy object via {@link #merge}, which merges + * the files. At the end of the ZIP combination, {@link #finish} is called, + * which then writes the merged single entry of that name. + * + * <p>Implementations of this interface are not required to be thread-safe. + * Thread-safety is achieved by creating multiple instances. Each instance + * that is separately passed to {@link StrategyCallback#customMerge} is + * guaranteed not to be called by two threads at the same time. + */ + interface CustomMergeStrategy { + + /** + * Merges another file into the current state. This method is called for + * every file entry of the same name. + */ + void merge(InputStream in, OutputStream out) throws IOException; + + /** + * Outputs the merged result into the given output stream. This method is + * only called once when no further file of the same name is available. + */ + void finish(OutputStream out) throws IOException; + } + + /** + * A callback interface for the {@link ZipEntryFilter#accept} method. Use + * this interface to indicate the type of processing for the given file name. + * For every file name, exactly one of the methods must be called once. A + * second method call throws {@link IllegalStateException}. + * + * <p>There is no guarantee that the callback will perform the requested + * operation at the time of the invocation. An implementation may choose to + * defer the operation to an arbitrary later time. + * + * <p>IMPORTANT NOTE: Do not implement this interface. It will be modified to + * support future extensions, and all implementations in this package will be + * updated. If you violate this advice, your code will break. + */ + interface StrategyCallback { + + /** + * Skips the current entry and all entries with the same name. + */ + void skip() throws IOException; + + /** + * Copies the current entry and skips all further entries with the same + * name. If {@code date} is non-null, then the timestamp of the entry is + * overwritten with the given value. + */ + void copy(Date date) throws IOException; + + /** + * Renames and copies the current entry, and skips all further entries with + * the same name. If {@code date} is non-null, then the timestamp of the entry + * is overwritten with the given value. + */ + void rename(String filename, Date date) throws IOException; + + /** + * Merges this and all further entries with the same name with the given + * {@link CustomMergeStrategy}. This method must never be called twice with + * the same object. If {@code date} is non-null, then the timestamp of the + * generated entry is set to the given value; otherwise, it is set to the + * current time. + */ + void customMerge(Date date, CustomMergeStrategy strategy) throws IOException; + } + + /** + * Determines the policy with which to handle the ZIP file entry with the + * given name and calls the appropriate method on the callback interface + * {@link StrategyCallback}. For every unique name in the set of all ZIP file + * entries, this method is called exactly once and the result is used for all + * entries of the same name. Except, if an entry is renamed, the original name + * is not considered as having been encountered yet. + * + * <p>Implementations should use the filename to distinguish the desired + * processing, call one method on the callback interface and return + * immediately after that call. + * + * <p>There is no guarantee that the callback will perform the requested + * operation at the time of the invocation. An implementation may choose to + * defer the operation to an arbitrary later time. + */ + void accept(String filename, StrategyCallback callback) throws IOException; +} |