diff options
Diffstat (limited to 'src/java_tools/buildjar/java/com/google/devtools/build/buildjar')
20 files changed, 3631 insertions, 0 deletions
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractJavaBuilder.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractJavaBuilder.java new file mode 100644 index 0000000000..15349a1789 --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractJavaBuilder.java @@ -0,0 +1,268 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.io.Files; +import com.google.devtools.build.buildjar.javac.JavacRunner; +import com.google.devtools.build.buildjar.javac.JavacRunnerImpl; + +import com.sun.tools.javac.main.Main.Result; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.util.List; +import java.util.zip.ZipEntry; + +/** + * A command line interface to compile a java_library rule using in-process + * javac. This allows us to spawn multiple java_library compilations on a + * single machine or distribute Java compilations to multiple machines. + */ +public abstract class AbstractJavaBuilder extends AbstractLibraryBuilder { + + /** The name of the protobuf meta file. */ + private static final String PROTOBUF_META_NAME = "protobuf.meta"; + + /** Enables more verbose output from the compiler. */ + protected boolean debug = false; + + @Override + protected boolean keepFileDuringCleanup(File file) { + return false; + } + + /** + * Flush the buffers of this JavaBuilder + */ + @SuppressWarnings("unused") // IOException + public synchronized void flush(OutputStream err) throws IOException { + } + + /** + * Shut this JavaBuilder down + */ + @SuppressWarnings("unused") // IOException + public synchronized void shutdown(OutputStream err) throws IOException { + } + + /** + * Prepares a compilation run and sets everything up so that the source files + * in the build request can be compiled. Invokes compileSources to do the + * actual compilation. + * + * @param build A JavaLibraryBuildRequest request object describing what to + * compile + * @param err PrintWriter for logging any diagnostic output + */ + public void compileJavaLibrary(final JavaLibraryBuildRequest build, final OutputStream err) + throws IOException { + prepareSourceCompilation(build); + + final String[] message = { null }; + final JavacRunner javacRunner = new JavacRunnerImpl(build.getPlugins()); + runWithLargeStack(new Runnable() { + @Override + public void run() { + try { + internalCompileJavaLibrary(build, javacRunner, err); + } catch (JavacException e) { + message[0] = e.getMessage(); + } catch (Exception e) { + // Some exceptions have a null message, yet the stack trace is useful + e.printStackTrace(); + message[0] = "java compilation threw exception: " + e.getMessage(); + } + } + }, 4L * 1024 * 1024); // 4MB stack + + if (message[0] != null) { + throw new IOException("Error compiling java source: " + message[0]); + } + } + + /** + * Compiles the java files of the java library specified in the build request.<p> + * The compilation consists of two parts:<p> + * First, javac is invoked directly to compile the java files in the build request.<p> + * Second, additional processing is done to the .class files that came out of the compile.<p> + * + * @param build A JavaLibraryBuildRequest request object describing what to compile + * @param err OutputStream for logging any diagnostic output + */ + private void internalCompileJavaLibrary(JavaLibraryBuildRequest build, JavacRunner javacRunner, + OutputStream err) throws IOException, JavacException { + // result may not be null, in case somebody changes the set of source files + // to the empty set + Result result = Result.OK; + if (!build.getSourceFiles().isEmpty()) { + PrintWriter javacErrorOutputWriter = new PrintWriter(err); + try { + result = compileSources(build, javacRunner, javacErrorOutputWriter); + } finally { + javacErrorOutputWriter.flush(); + } + } + + if (!result.isOK()) { + throw new JavacException(result); + } + runClassPostProcessing(build); + } + + /** + * Build a jar file containing source files that were generated by an annotation processor. + */ + public abstract void buildGensrcJar(JavaLibraryBuildRequest build, OutputStream err) + throws IOException; + + @VisibleForTesting + protected void runClassPostProcessing(JavaLibraryBuildRequest build) + throws IOException { + for (AbstractPostProcessor postProcessor : build.getPostProcessors()) { + postProcessor.initialize(build); + postProcessor.processRequest(); + } + } + + /** + * Compiles the java files specified in 'JavaLibraryBuildRequest'. + * Implementations can try to avoid recompiling the java files. Whenever + * this function is invoked, it is guaranteed that the build request + * contains files to compile. + * + * @param build A JavaLibraryBuildRequest request object describing what to + * compile + * @param err PrintWriter for logging any diagnostic output + * @return the exit status of the java compiler. + */ + abstract Result compileSources(JavaLibraryBuildRequest build, JavacRunner javacRunner, + PrintWriter err) throws IOException; + + /** + * Perform the build. + */ + public void run(JavaLibraryBuildRequest build, PrintStream err) + throws IOException { + boolean successful = false; + try { + compileJavaLibrary(build, err); + buildJar(build); + if (!build.getProcessors().isEmpty()) { + if (build.getGeneratedSourcesOutputJar() != null) { + buildGensrcJar(build, err); + } + } + successful = true; + } finally { + build.getDependencyModule().emitUsedClasspath(build.getClassPath()); + build.getDependencyModule().emitDependencyInformation(build.getClassPath(), successful); + shutdown(err); + } + } + + // Utility functions + + /** + * Runs "run" in another thread (whose lifetime is contained within the + * activation of this function call) using a stack size of 'stackSize' bytes. + * Unchecked exceptions thrown by the Runnable will be re-thrown in the main + * thread. + */ + private static void runWithLargeStack(final Runnable run, long stackSize) { + final Throwable[] unchecked = { null }; + Thread t = new Thread(null, run, "runWithLargeStack", stackSize); + t.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + unchecked[0] = e; + } + }); + t.start(); + boolean wasInterrupted = false; + for (;;) { + try { + t.join(0); + break; + } catch (InterruptedException e) { + wasInterrupted = true; + } + } + if (wasInterrupted) { + Thread.currentThread().interrupt(); + } + if (unchecked[0] instanceof Error) { + throw (Error) unchecked[0]; + } else if (unchecked[0] instanceof RuntimeException) { + throw (RuntimeException) unchecked[0]; + } + } + + /** + * A SourceJarEntryListener that collects protobuf meta data files from the + * source jar files. + */ + private static class ProtoMetaFileCollector implements SourceJarEntryListener { + + private final String sourceDir; + private final String outputDir; + private final ByteArrayOutputStream buffer; + + public ProtoMetaFileCollector(String sourceDir, String outputDir) { + this.sourceDir = sourceDir; + this.outputDir = outputDir; + this.buffer = new ByteArrayOutputStream(); + } + + @Override + public void onEntry(ZipEntry entry) throws IOException { + String entryName = entry.getName(); + if (!entryName.equals(PROTOBUF_META_NAME)) { + return; + } + Files.copy(new File(sourceDir, PROTOBUF_META_NAME), buffer); + } + + /** + * Writes the combined the meta files into the output directory. Delete the + * stalling meta file if no meta file is collected. + */ + @Override + public void finish() throws IOException { + File outputFile = new File(outputDir, PROTOBUF_META_NAME); + if (buffer.size() > 0) { + try (OutputStream outputStream = new FileOutputStream(outputFile)) { + buffer.writeTo(outputStream); + } + } else if (outputFile.exists()) { + // Delete stalled meta file. + outputFile.delete(); + } + } + } + + @Override + protected List<SourceJarEntryListener> getSourceJarEntryListeners( + JavaLibraryBuildRequest build) { + List<SourceJarEntryListener> result = super.getSourceJarEntryListeners(build); + result.add(new ProtoMetaFileCollector( + build.getTempDir(), build.getClassDir())); + return result; + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractLibraryBuilder.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractLibraryBuilder.java new file mode 100644 index 0000000000..cf1c985b0f --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractLibraryBuilder.java @@ -0,0 +1,295 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.io.ByteStreams; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Enumeration; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * Base class for java_library builders. + * + * <p>Implements common functionality like source files preparation and + * output jar creation. + */ +public abstract class AbstractLibraryBuilder extends CommonJavaLibraryProcessor { + + /** + * Prepares a compilation run. This involves cleaning up temporary dircectories and + * writing the classpath files. + */ + protected void prepareSourceCompilation(JavaLibraryBuildRequest build) throws IOException { + File classDirectory = new File(build.getClassDir()); + if (classDirectory.exists()) { + try { + // Necessary for local builds in order to discard previous outputs + cleanupOutputDirectory(classDirectory); + } catch (IOException e) { + throw new IOException("Cannot clean output directory '" + classDirectory + "'", e); + } + } + classDirectory.mkdirs(); + + setUpSourceJars(build); + } + + public void buildJar(JavaLibraryBuildRequest build) throws IOException { + JarCreator jar = new JarCreator(build.getOutputJar()); + jar.setNormalize(true); + jar.setCompression(build.compressJar()); + + // The easiest way to handle resource jars is to unpack them into the class directory, just + // before we start zipping it up. + for (String resourceJar : build.getResourceJars()) { + setUpSourceJar(new File(resourceJar), build.getClassDir(), + new ArrayList<SourceJarEntryListener>()); + } + + jar.addDirectory(build.getClassDir()); + + jar.addRootEntries(build.getRootResourceFiles()); + addResourceEntries(jar, build.getResourceFiles()); + addMessageEntries(jar, build.getMessageFiles()); + + jar.execute(); + } + + /** + * Adds a collection of resource entries. Each entry is a string composed of a + * pair of parts separated by a colon ':'. The name of the resource comes from + * the second part, and the path to the resource comes from the whole string + * with the colon replaced by a slash '/'. + * <pre> + * prefix:name => (name, prefix/name) + * </pre> + */ + private static void addResourceEntries(JarCreator jar, Collection<String> resources) + throws IOException { + for (String resource : resources) { + int colon = resource.indexOf(':'); + if (colon < 0) { + throw new IOException("" + resource + ": Illegal resource entry."); + } + String prefix = resource.substring(0, colon); + String name = resource.substring(colon + 1); + String path = colon > 0 ? prefix + "/" + name : name; + addEntryWithParents(jar, name, path); + } + } + + private static void addMessageEntries(JarCreator jar, List<String> messages) + throws IOException { + for (String message : messages) { + int colon = message.indexOf(':'); + if (colon < 0) { + throw new IOException("" + message + ": Illegal message entry."); + } + String prefix = message.substring(0, colon); + String name = message.substring(colon + 1); + String path = colon > 0 ? prefix + "/" + name : name; + File messageFile = new File(path); + // Ignore empty messages. They get written by the translation importer + // when there is no translation for a particular language. + if (messageFile.length() != 0L) { + addEntryWithParents(jar, name, path); + } + } + } + + /** + * Adds an entry to the jar, making sure that all the parent dirs up to the + * base of {@code entry} are also added. + * + * @param entry the PathFragment of the entry going into the Jar file + * @param file the PathFragment of the input file for the entry + */ + @VisibleForTesting + static void addEntryWithParents(JarCreator creator, String entry, String file) { + while ((entry != null) && creator.addEntry(entry, file)) { + entry = new File(entry).getParent(); + file = new File(file).getParent(); + } + } + + /** + * Internal interface which will listen on each entry of the source jar + * files during the source jar setup process. + */ + protected interface SourceJarEntryListener { + void onEntry(ZipEntry entry) throws IOException; + void finish() throws IOException; + } + + protected List<SourceJarEntryListener> getSourceJarEntryListeners(JavaLibraryBuildRequest build) { + List<SourceJarEntryListener> result = new ArrayList<>(); + result.add(new SourceJavaFileCollector(build)); + return result; + } + + /** + * A SourceJarEntryListener that collects a lists of source Java files from + * the source jar files. + */ + private static class SourceJavaFileCollector implements SourceJarEntryListener { + private final List<String> sources; + private final JavaLibraryBuildRequest build; + + public SourceJavaFileCollector(JavaLibraryBuildRequest build) { + this.sources = new ArrayList<>(); + this.build = build; + } + + @Override + public void onEntry(ZipEntry entry) { + String entryName = entry.getName(); + if (entryName.endsWith(".java")) { + sources.add(build.getTempDir() + File.separator + entryName); + } + } + + @Override + public void finish() { + build.getSourceFiles().addAll(sources); + } + } + + /** + * Extracts the all source jars from the build request into the temporary + * directory specified in the build request. Empties the temporary directory, + * if it exists. + */ + private void setUpSourceJars(JavaLibraryBuildRequest build) throws IOException { + String sourcesDir = build.getTempDir(); + + File sourceDirFile = new File(sourcesDir); + if (sourceDirFile.exists()) { + cleanupDirectory(sourceDirFile, true); + } + + if (build.getSourceJars().isEmpty()) { + return; + } + + List<SourceJarEntryListener> listeners = getSourceJarEntryListeners(build); + for (String sourceJar : build.getSourceJars()) { + setUpSourceJar(new File(sourceJar), sourcesDir, listeners); + } + for (SourceJarEntryListener listener : listeners) { + listener.finish(); + } + } + + /** + * Extracts the source jar into the directory sourceDir. Calls each of the + * SourceJarEntryListeners for each non-directory entry to do additional work. + */ + private void setUpSourceJar(File sourceJar, String sourceDir, + List<SourceJarEntryListener> listeners) + throws IOException { + try (ZipFile zipFile = new ZipFile(sourceJar)) { + Enumeration<? extends ZipEntry> zipEntries = zipFile.entries(); + while (zipEntries.hasMoreElements()) { + ZipEntry currentEntry = zipEntries.nextElement(); + String entryName = currentEntry.getName(); + File outputFile = new File(sourceDir, entryName); + + outputFile.getParentFile().mkdirs(); + + if (currentEntry.isDirectory()) { + outputFile.mkdir(); + } else { + // Copy the data from the zip file to the output file. + try (InputStream in = zipFile.getInputStream(currentEntry); + OutputStream out = new FileOutputStream(outputFile)) { + ByteStreams.copy(in, out); + } + + for (SourceJarEntryListener listener : listeners) { + listener.onEntry(currentEntry); + } + } + } + } + } + + /** + * Recursively cleans up the files beneath the specified output directory. + * Does not follow symbolic links. Throws IOException if any deletion fails. + * + * Will delete all empty directories. + * + * @param dir the directory to clean up. + * @return true if the directory itself was removed as well. + */ + boolean cleanupOutputDirectory(File dir) throws IOException { + return cleanupDirectory(dir, false); + } + + /** + * Recursively cleans up the files beneath the specified output directory. + * Does not follow symbolic links. Throws IOException if any deletion fails. + * If removeEverything is false, keeps .class files if keepClassFilesDuringCleanup() + * returns true, and also keeps all flags.xml files. + * If removeEverything is true, removes everything. + * Will delete all empty directories. + * + * @param dir the directory to clean up. + * @param removeEverything whether to remove all files, or keep flags.xml/.class files. + * @return true if the directory itself was removed as well. + */ + private boolean cleanupDirectory(File dir, boolean removeEverything) throws IOException { + boolean isEmpty = true; + File[] files = dir.listFiles(); + if (files == null) { return false; } // avoid race condition + for (File file : files) { + if (file.isDirectory()) { + isEmpty &= cleanupDirectory(file, removeEverything); + } else if (!removeEverything && keepClassFilesDuringCleanup() && + file.getName().endsWith(".class")) { + isEmpty = false; + } else if (!removeEverything && keepFileDuringCleanup(file)) { + isEmpty = false; + } else { + file.delete(); + } + } + if (isEmpty) { + dir.delete(); + } + return isEmpty; + } + + protected abstract boolean keepFileDuringCleanup(File file); + + /** + * Returns true if cleaning the output directory should remove all + * .class files in the output directory. + */ + protected boolean keepClassFilesDuringCleanup() { + return false; + } + +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractPostProcessor.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractPostProcessor.java new file mode 100644 index 0000000000..e8e608d44b --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractPostProcessor.java @@ -0,0 +1,119 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar; + +import com.google.common.base.Preconditions; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * A processor to apply additional steps to the compiled java classes. It can be used to add code + * coverage instrumentation for instance. + */ +public abstract class AbstractPostProcessor { + private static final Map<String, AbstractPostProcessor> postProcessors = new HashMap<>(); + + /** + * Declares a post processor with a name. This name serves as the command line argument to + * reference a processor. + * + * @param name the command line name of the processor + * @param postProcessor the post processor object + */ + public static void addPostProcessor(String name, AbstractPostProcessor postProcessor) { + postProcessors.put(name, postProcessor); + } + + private String workingDir = null; + private JavaLibraryBuildRequest build = null; + + /** + * Sets the command line arguments for this processor. + * + * @param arguments the list of arguments + * + * @throws InvalidCommandLineException when the list of arguments for this processors is + * incorrect. + */ + public abstract void setCommandLineArguments(List<String> arguments) + throws InvalidCommandLineException; + + /** + * This initializer is outside of the constructor so the arguments are not passed to the + * descendants. + */ + void initialize(JavaLibraryBuildRequest build) { + this.build = build; + } + + protected String workingPath(String name) { + Preconditions.checkNotNull(this.build); + return workingDir != null && name.length() > 0 && name.charAt(0) != '/' + ? workingDir + File.separator + name + : name; + } + + protected boolean shouldCompressJar() { + return build.compressJar(); + } + + protected String getBuildClassDir() { + return build.getClassDir(); + } + + /** + * Main interface method of the post processor. + */ + public abstract void processRequest() throws IOException; + + /** + * Create an {@link AbstractPostProcessor} using reflection. + * + * @param processorName the name of the processor to instantiate. It should exist in the list of + * post processors added with the {@link #addPostProcessor(String, AbstractPostProcessor)} + * method. + * @param arguments the list of arguments that should be passed to the processor during + * instantiation. + * @throws InvalidCommandLineException on error creating the processor + */ + static AbstractPostProcessor create(String processorName, List<String> arguments) + throws InvalidCommandLineException { + AbstractPostProcessor processor = postProcessors.get(processorName); + if (processor == null) { + throw new InvalidCommandLineException("No such processor '" + processorName + "'"); + } + processor.setCommandLineArguments(arguments); + return processor; + } + + /** + * Recursively delete the given file, it is unsafe. + * + * @param file the file to recursively remove + */ + protected static void recursiveRemove(File file) { + if (file.isDirectory()) { + for (File f : file.listFiles()) { + recursiveRemove(f); + } + } else if (file.exists()) { + file.delete(); + } + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/BazelJavaBuilder.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/BazelJavaBuilder.java new file mode 100644 index 0000000000..4cd1dfe748 --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/BazelJavaBuilder.java @@ -0,0 +1,42 @@ +// Copyright 2007-2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar; + +import java.io.IOException; +import java.util.Arrays; + +/** + * The JavaBuilder main called by bazel. + */ +public abstract class BazelJavaBuilder { + + private static final String CMDNAME = "BazelJavaBuilder"; + + /** + * The main method of the BazelJavaBuilder. + */ + public static void main(String[] args) { + try { + JavaLibraryBuildRequest build = JavaLibraryBuildRequest.parse(Arrays.asList(args)); + AbstractJavaBuilder builder = build.getDependencyModule().reduceClasspath() + ? new ReducedClasspathJavaLibraryBuilder() + : new SimpleJavaLibraryBuilder(); + builder.run(build, System.err); + } catch (IOException | InvalidCommandLineException e) { + System.err.println(CMDNAME + " threw exception : " + e.getMessage()); + System.exit(1); + } + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/CommonJavaLibraryProcessor.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/CommonJavaLibraryProcessor.java new file mode 100644 index 0000000000..dae8d9715b --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/CommonJavaLibraryProcessor.java @@ -0,0 +1,65 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar; + +import com.sun.tools.javac.main.Main.Result; + +import java.util.ArrayList; +import java.util.List; + +/** + * Superclass for all JavaBuilder processor classes + * involved in compiling and processing java code. + */ +public abstract class CommonJavaLibraryProcessor { + + /** + * Exception used to represent failed javac invocation. + */ + static final class JavacException extends Exception { + public JavacException(Result result) { + super("java compilation returned status " + result); + if (result.isOK()) { + throw new IllegalArgumentException(); + } + } + } + + /** + * Creates the initial set of arguments to javac from the Build + * configuration supplied. This set of arguments should be extended + * by the code invoking it. + * + * @param build The build request for the initial set of arguments is needed + * @return The list of initial arguments + */ + protected List<String> createInitialJavacArgs(JavaLibraryBuildRequest build, + String classPath) { + List<String> args = new ArrayList<>(); + if (!classPath.isEmpty()) { + args.add("-cp"); + args.add(classPath); + } + args.add("-d"); + args.add(build.getClassDir()); + + // Add an empty source path to prevent javac from sucking in source files + // from .jar files on the classpath. + args.add("-sourcepath"); + args.add(":"); + + return args; + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/InvalidCommandLineException.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/InvalidCommandLineException.java new file mode 100644 index 0000000000..a84c460c69 --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/InvalidCommandLineException.java @@ -0,0 +1,29 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar; + +/** + * Exception to be thrown on command line parsing errors + */ +public class InvalidCommandLineException extends Exception { + + public InvalidCommandLineException(String message) { + super(message); + } + + public InvalidCommandLineException(String message, Throwable cause) { + super(message, cause); + } +}
\ No newline at end of file diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JarCreator.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JarCreator.java new file mode 100644 index 0000000000..ead7ceb3dc --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JarCreator.java @@ -0,0 +1,200 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.TreeMap; +import java.util.jar.Attributes; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; + +/** + * A class for creating Jar files. Allows normalization of Jar entries by setting their timestamp to + * the DOS epoch. All Jar entries are sorted alphabetically. + */ +public class JarCreator extends JarHelper { + + // Map from Jar entry names to files. Use TreeMap so we can establish a canonical order for the + // entries regardless in what order they get added. + private final Map<String, String> jarEntries = new TreeMap<>(); + private String manifestFile; + private String mainClass; + + public JarCreator(String fileName) { + super(fileName); + } + + /** + * Adds an entry to the Jar file, normalizing the name. + * + * @param entryName the name of the entry in the Jar file + * @param fileName the name of the input file for the entry + * @return true iff a new entry was added + */ + public boolean addEntry(String entryName, String fileName) { + if (entryName.startsWith("/")) { + entryName = entryName.substring(1); + } else if (entryName.startsWith("./")) { + entryName = entryName.substring(2); + } + return jarEntries.put(entryName, fileName) == null; + } + + /** + * Adds the contents of a directory to the Jar file. All files below this + * directory will be added to the Jar file using the name relative to the + * directory as the name for the Jar entry. + * + * @param directory the directory to add to the jar + */ + public void addDirectory(String directory) { + addDirectory(null, new File(directory)); + } + + /** + * Adds the contents of a directory to the Jar file. All files below this + * directory will be added to the Jar file using the prefix and the name + * relative to the directory as the name for the Jar entry. Always uses '/' as + * the separator char for the Jar entries. + * + * @param prefix the prefix to prepend to every Jar entry name found below the + * directory + * @param directory the directory to add to the Jar + */ + private void addDirectory(String prefix, File directory) { + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + String entryName = prefix != null ? prefix + "/" + file.getName() : file.getName(); + jarEntries.put(entryName, file.getAbsolutePath()); + if (file.isDirectory()) { + addDirectory(entryName, file); + } + } + } + } + + /** + * Adds a collection of entries to the jar, each with a given source path, and with + * the resulting file in the root of the jar. + * <pre> + * some/long/path.foo => (path.foo, some/long/path.foo) + * </pre> + */ + public void addRootEntries(Collection<String> entries) { + for (String entry : entries) { + jarEntries.put(new File(entry).getName(), entry); + } + } + + /** + * Sets the main.class entry for the manifest. A value of <code>null</code> + * (the default) will omit the entry. + * + * @param mainClass the fully qualified name of the main class + */ + public void setMainClass(String mainClass) { + this.mainClass = mainClass; + } + + /** + * Sets filename for the manifest content. If this is set the manifest will be + * read from this file otherwise the manifest content will get generated on + * the fly. + * + * @param manifestFile the filename of the manifest file. + */ + public void setManifestFile(String manifestFile) { + this.manifestFile = manifestFile; + } + + private byte[] manifestContent() throws IOException { + Manifest manifest; + if (manifestFile != null) { + FileInputStream in = new FileInputStream(manifestFile); + manifest = new Manifest(in); + } else { + manifest = new Manifest(); + } + Attributes attributes = manifest.getMainAttributes(); + attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); + Attributes.Name createdBy = new Attributes.Name("Created-By"); + if (attributes.getValue(createdBy) == null) { + attributes.put(createdBy, "blaze"); + } + if (mainClass != null) { + attributes.put(Attributes.Name.MAIN_CLASS, mainClass); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + manifest.write(out); + return out.toByteArray(); + } + + /** + * Executes the creation of the Jar file. + * + * @throws IOException if the Jar cannot be written or any of the entries + * cannot be read. + */ + public void execute() throws IOException { + out = new JarOutputStream(new BufferedOutputStream(new FileOutputStream(jarFile))); + + // Create the manifest entry in the Jar file + writeManifestEntry(manifestContent()); + try { + for (Map.Entry<String, String> entry : jarEntries.entrySet()) { + copyEntry(entry.getKey(), new File(entry.getValue())); + } + } finally { + out.closeEntry(); + out.close(); + } + } + + /** + * A simple way to create Jar file using the JarCreator class. + */ + public static void main(String[] args) { + if (args.length < 1) { + System.err.println("usage: CreateJar output [root directories]"); + System.exit(1); + } + String output = args[0]; + JarCreator createJar = new JarCreator(output); + for (int i = 1; i < args.length; i++) { + createJar.addDirectory(args[i]); + } + createJar.setCompression(true); + createJar.setNormalize(true); + createJar.setVerbose(true); + long start = System.currentTimeMillis(); + try { + createJar.execute(); + } catch (IOException e) { + e.printStackTrace(); + System.err.println(e.getMessage()); + System.exit(1); + } + long stop = System.currentTimeMillis(); + System.err.println((stop - start) + "ms."); + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JarHelper.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JarHelper.java new file mode 100644 index 0000000000..0832082707 --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JarHelper.java @@ -0,0 +1,201 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar; + +import com.google.common.hash.Hashing; +import com.google.common.io.Files; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; + +/** + * A simple helper class for creating Jar files. All Jar entries are sorted alphabetically. Allows + * normalization of Jar entries by setting the timestamp of non-.class files to the DOS epoch. + * Timestamps of .class files are set to the DOS epoch + 2 seconds (The zip timestamp granularity) + * Adjusting the timestamp for .class files is neccessary since otherwise javac will recompile java + * files if both the java file and its .class file are present. + */ +public class JarHelper { + + public static final String MANIFEST_DIR = "META-INF/"; + public static final String MANIFEST_NAME = JarFile.MANIFEST_NAME; + public static final String SERVICES_DIR = "META-INF/services/"; + + public static final long DOS_EPOCH_IN_JAVA_TIME = 315561600000L; + + // ZIP timestamps have a resolution of 2 seconds. + // see http://www.info-zip.org/FAQ.html#limits + public static final long MINIMUM_TIMESTAMP_INCREMENT = 2000L; + + // The name of the Jar file we want to create + protected final String jarFile; + + // The properties to describe how to create the Jar + protected boolean normalize; + protected int storageMethod = JarEntry.DEFLATED; + protected boolean verbose = false; + + // The state needed to create the Jar + protected final Set<String> names = new HashSet<>(); + protected JarOutputStream out; + + public JarHelper(String filename) { + jarFile = filename; + } + + /** + * Enables or disables the Jar entry normalization. + * + * @param normalize If true the timestamps of Jar entries will be set to the + * DOS epoch. + */ + public void setNormalize(boolean normalize) { + this.normalize = normalize; + } + + /** + * Enables or disables compression for the Jar file entries. + * + * @param compression if true enables compressions for the Jar file entries. + */ + public void setCompression(boolean compression) { + storageMethod = compression ? JarEntry.DEFLATED : JarEntry.STORED; + } + + /** + * Enables or disables verbose messages. + * + * @param verbose if true enables verbose messages. + */ + public void setVerbose(boolean verbose) { + this.verbose = verbose; + } + + /** + * Returns the normalized timestamp for a jar entry based on its name. + * This is necessary since javac will, when loading a class X, prefer a + * source file to a class file, if both files have the same timestamp. + * Therefore, we need to adjust the timestamp for class files to slightly + * after the normalized time. + * @param name The name of the file for which we should return the + * normalized timestamp. + * @return the time for a new Jar file entry in milliseconds since the epoch. + */ + private long normalizedTimestamp(String name) { + if (name.endsWith(".class")) { + return DOS_EPOCH_IN_JAVA_TIME + MINIMUM_TIMESTAMP_INCREMENT; + } else { + return DOS_EPOCH_IN_JAVA_TIME; + } + } + + /** + * Returns the time for a new Jar file entry in milliseconds since the epoch. + * Uses {@link JarCreator#DOS_EPOCH_IN_JAVA_TIME} for normalized entries, + * {@link System#currentTimeMillis()} otherwise. + * + * @param filename The name of the file for which we are entering the time + * @return the time for a new Jar file entry in milliseconds since the epoch. + */ + protected long newEntryTimeMillis(String filename) { + return normalize ? normalizedTimestamp(filename) : System.currentTimeMillis(); + } + + /** + * Writes an entry with specific contents to the jar. Directory entries must + * include the trailing '/'. + */ + protected void writeEntry(JarOutputStream out, String name, byte[] content) throws IOException { + if (names.add(name)) { + // Create a new entry + JarEntry entry = new JarEntry(name); + entry.setTime(newEntryTimeMillis(name)); + int size = content.length; + entry.setSize(size); + if (size == 0) { + entry.setMethod(JarEntry.STORED); + entry.setCrc(0); + out.putNextEntry(entry); + } else { + entry.setMethod(storageMethod); + if (storageMethod == JarEntry.STORED) { + entry.setCrc(Hashing.crc32().hashBytes(content).padToLong()); + } + out.putNextEntry(entry); + out.write(content); + } + out.closeEntry(); + } + } + + /** + * Writes a standard Java manifest entry into the JarOutputStream. This + * includes the directory entry for the "META-INF" directory + * + * @param content the Manifest content to write to the manifest entry. + * @throws IOException + */ + protected void writeManifestEntry(byte[] content) throws IOException { + writeEntry(out, MANIFEST_DIR, new byte[]{}); + writeEntry(out, MANIFEST_NAME, content); + } + + /** + * Copies file or directory entries from the file system into the jar. + * Directory entries will be detected and their names automatically '/' + * suffixed. + */ + protected void copyEntry(String name, File file) throws IOException { + if (!names.contains(name)) { + if (!file.exists()) { + throw new FileNotFoundException(file.getAbsolutePath() + " (No such file or directory)"); + } + boolean isDirectory = file.isDirectory(); + if (isDirectory && !name.endsWith("/")) { + name = name + '/'; // always normalize directory names before checking set + } + if (names.add(name)) { + if (verbose) { + System.err.println("adding " + file); + } + // Create a new entry + long size = isDirectory ? 0 : file.length(); + JarEntry outEntry = new JarEntry(name); + long newtime = normalize ? normalizedTimestamp(name) : file.lastModified(); + outEntry.setTime(newtime); + outEntry.setSize(size); + if (size == 0L) { + outEntry.setMethod(JarEntry.STORED); + outEntry.setCrc(0); + out.putNextEntry(outEntry); + } else { + outEntry.setMethod(storageMethod); + if (storageMethod == JarEntry.STORED) { + outEntry.setCrc(Files.hash(file, Hashing.crc32()).padToLong()); + } + out.putNextEntry(outEntry); + Files.copy(file, out); + } + out.closeEntry(); + } + } + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JavaLibraryBuildRequest.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JavaLibraryBuildRequest.java new file mode 100644 index 0000000000..9b899e50ef --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JavaLibraryBuildRequest.java @@ -0,0 +1,516 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin; +import com.google.devtools.build.buildjar.javac.plugins.dependency.DependencyModule; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.NoSuchElementException; + +/** + * All the information needed to perform a single Java library build operation. + */ +public final class JavaLibraryBuildRequest { + private boolean compressJar; + + private final List<String> sourceFiles; + private final ImmutableList<String> sourceJars; + private final ImmutableList<String> messageFiles; + private final ImmutableList<String> resourceFiles; + private final ImmutableList<String> resourceJars; + /** Resource files that should be put in the root of the output jar. */ + private final ImmutableList<String> rootResourceFiles; + + private final String classPath; + + private final String processorPath; + private final List<String> processorNames; + + private final String classDir; + private final String tempDir; + + private final String outputJar; + + // Post processors + private final ImmutableList<AbstractPostProcessor> postProcessors; + + private final ImmutableList<String> javacOpts; + + /** + * Where to store source files generated by annotation processors. + */ + private final String sourceGenDir; + + /** + * The path to an output jar for source files generated by annotation processors. + */ + private final String generatedSourcesOutputJar; + + /** + * Repository for all dependency-related information. + */ + private final DependencyModule dependencyModule; + + /** + * List of plugins that are given to javac. + */ + private final ImmutableList<BlazeJavaCompilerPlugin> plugins; + + private JavaLibraryBuildRequest( + boolean compressJar, + List<String> sourceFiles, ImmutableList<String> sourceJars, + ImmutableList<String> messageFiles, ImmutableList<String> resourceFiles, + ImmutableList<String> resourceJars, ImmutableList<String> rootResourceFiles, + String classPath, String processorPath, List<String> processorNames, String classDir, + String tempDir, String outputJar, ImmutableList<AbstractPostProcessor> postProcessors, + ImmutableList<String> javacOpts, String sourceGenDir, String generatedSourcesOutputJar, + DependencyModule dependencyModule, ImmutableList<BlazeJavaCompilerPlugin> plugins) { + this.compressJar = compressJar; + this.sourceFiles = sourceFiles; + this.sourceJars = sourceJars; + this.messageFiles = messageFiles; + this.resourceFiles = resourceFiles; + this.resourceJars = resourceJars; + this.rootResourceFiles = rootResourceFiles; + this.classPath = classPath; + this.processorPath = processorPath; + this.processorNames = processorNames; + this.classDir = classDir; + this.tempDir = tempDir; + this.outputJar = outputJar; + this.postProcessors = postProcessors; + this.javacOpts = javacOpts; + this.sourceGenDir = sourceGenDir; + this.generatedSourcesOutputJar = generatedSourcesOutputJar; + this.dependencyModule = dependencyModule; + this.plugins = plugins; + } + + public boolean compressJar() { + return compressJar; + } + + public List<String> getSourceFiles() { + // TODO(bazel-team): This is being modified after parsing to add files from source jars. + return sourceFiles; + } + + public ImmutableList<String> getSourceJars() { + return sourceJars; + } + + public ImmutableList<String> getMessageFiles() { + return messageFiles; + } + + public ImmutableList<String> getResourceFiles() { + return resourceFiles; + } + + public ImmutableList<String> getResourceJars() { + return resourceJars; + } + + public ImmutableList<String> getRootResourceFiles() { + return rootResourceFiles; + } + + public String getClassPath() { + return classPath; + } + + public String getProcessorPath() { + return processorPath; + } + + public List<String> getProcessors() { + // TODO(bazel-team): This might be modified by a JavaLibraryBuilder to enable specific + // annotation processors. + return processorNames; + } + + public String getClassDir() { + return classDir; + } + + public String getTempDir() { + return tempDir; + } + + public String getOutputJar() { + return outputJar; + } + + public ImmutableList<String> getJavacOpts() { + return javacOpts; + } + + public String getSourceGenDir() { + return sourceGenDir; + } + + public String getGeneratedSourcesOutputJar() { + return generatedSourcesOutputJar; + } + + public ImmutableList<AbstractPostProcessor> getPostProcessors() { + return postProcessors; + } + + public ImmutableList<BlazeJavaCompilerPlugin> getPlugins() { + return plugins; + } + + public DependencyModule getDependencyModule() { + return dependencyModule; + } + + /** + * Parses the list of arguments into a {@link JavaLibraryBuildRequest}. The returned + * {@link JavaLibraryBuildRequest} object can be then used to configure the compilation itself. + * + * @throws IOException if the argument list contains file (with the @ prefix) and reading that + * file failed. + * @throws InvalidCommandLineException on any command line error + */ + public static JavaLibraryBuildRequest parse(List<String> args) throws IOException, + InvalidCommandLineException { + return new JavaLibraryBuildRequest.Builder(args).build(); + } + + /** + * Builds a {@link JavaLibraryBuildRequest}. + */ + public static final class Builder { + private boolean compressJar; + + private final ImmutableList.Builder<String> sourceFiles = ImmutableList.builder(); + private final ImmutableList.Builder<String> sourceJars = ImmutableList.builder(); + private final ImmutableList.Builder<String> messageFiles = ImmutableList.builder(); + private final ImmutableList.Builder<String> resourceFiles = ImmutableList.builder(); + private final ImmutableList.Builder<String> resourceJars = ImmutableList.builder(); + private final ImmutableList.Builder<String> rootResourceFiles = ImmutableList.builder(); + + private String classPath; + + private String processorPath = ""; + private final List<String> processorNames = new ArrayList<>(); + + // Since the default behavior of this tool with no arguments is + // "rm -fr <classDir>", let's not default to ".", shall we? + private String classDir = "classes"; + private String tempDir = "_tmp"; + + private String outputJar; + + private final ImmutableList.Builder<AbstractPostProcessor> postProcessors = + ImmutableList.builder(); + + private String ruleKind; + private String targetLabel; + + private ImmutableList.Builder<String> javacOpts = ImmutableList.builder(); + + private String sourceGenDir; + + private String generatedSourcesOutputJar; + + private final DependencyModule dependencyModule; + + private final ImmutableList.Builder<BlazeJavaCompilerPlugin> plugins = ImmutableList.builder(); + + /** + * Constructs a build from a list of command args. Sets the same JavacRunner + * for both compilation and annotation processing. + * + * @param args the list of command line args + * @throws InvalidCommandLineException on any command line error + */ + public Builder(List<String> args) throws InvalidCommandLineException, IOException { + dependencyModule = processCommandlineArgs(expandArguments(args)); + plugins.add(dependencyModule.getPlugin()); + } + + /** + * Constructs a build from a list of command args. Sets the same JavacRunner + * for both compilation and annotation processing. + * + * @param args the list of command line args + * @param extraPlugins extraneous plugins to use in addition to the strict dependency module. + * @throws InvalidCommandLineException on any command line error + */ + public Builder(List<String> args, List<BlazeJavaCompilerPlugin> extraPlugins) + throws InvalidCommandLineException, IOException { + this(args); + plugins.addAll(extraPlugins); + } + + public ImmutableList<String> getJavacOpts() { + return javacOpts.build(); + } + + public void setJavacOpts(List<String> javacOpts) { + this.javacOpts = ImmutableList.<String>builder().addAll(javacOpts); + } + + public JavaLibraryBuildRequest build() { + ArrayList<String> sourceFiles = new ArrayList<>(this.sourceFiles.build()); + ImmutableList<String> sourceJars = this.sourceJars.build(); + ImmutableList<String> messageFiles = this.messageFiles.build(); + ImmutableList<String> resourceFiles = this.resourceFiles.build(); + ImmutableList<String> resourceJars = this.resourceJars.build(); + ImmutableList<String> rootResourceFiles = this.rootResourceFiles.build(); + ImmutableList<AbstractPostProcessor> postProcessors = this.postProcessors.build(); + ImmutableList<String> javacOpts = this.javacOpts.build(); + ImmutableList<BlazeJavaCompilerPlugin> plugins = this.plugins.build(); + return new JavaLibraryBuildRequest(compressJar, sourceFiles, sourceJars, messageFiles, + resourceFiles, resourceJars, rootResourceFiles, classPath, processorPath, processorNames, + classDir, tempDir, outputJar, postProcessors, javacOpts, sourceGenDir, + generatedSourcesOutputJar, dependencyModule, plugins); + } + + /** + * Processes the command line arguments. + * + * @throws InvalidCommandLineException on an invalid option being passed. + */ + private DependencyModule processCommandlineArgs(Deque<String> argQueue) + throws InvalidCommandLineException { + DependencyModule.Builder builder = new DependencyModule.Builder(); + for (String arg = argQueue.pollFirst(); arg != null; arg = argQueue.pollFirst()) { + switch (arg) { + case "--javacopts": + // Collect additional arguments to javac. + // Assumes that javac options do not start with "--". + // otherwise we have to do something like adding a "--" + // terminator to the passed arguments. + collectFlagArguments(javacOpts, argQueue, "--"); + break; + case "--direct_dependency": { + String jar = getArgument(argQueue, arg); + String target = getArgument(argQueue, arg); + builder.addDirectMapping(jar, target); + break; + } + case "--indirect_dependency": { + String jar = getArgument(argQueue, arg); + String target = getArgument(argQueue, arg); + builder.addIndirectMapping(jar, target); + break; + } + case "--strict_java_deps": + builder.setStrictJavaDeps(getArgument(argQueue, arg)); + break; + case "--output_deps": + builder.setOutputDepsFile(getArgument(argQueue, arg)); + break; + case "--output_deps_proto": + builder.setOutputDepsProtoFile(getArgument(argQueue, arg)); + break; + case "--deps_artifacts": + ImmutableList.Builder<String> depsArtifacts = ImmutableList.builder(); + collectFlagArguments(depsArtifacts, argQueue, "--"); + builder.addDepsArtifacts(depsArtifacts.build()); + break; + case "--reduce_classpath": + builder.setReduceClasspath(); + break; + case "--sourcegendir": + sourceGenDir = getArgument(argQueue, arg); + break; + case "--generated_sources_output": + generatedSourcesOutputJar = getArgument(argQueue, arg); + break; + default: + processArg(arg, argQueue); + } + } + builder.setRuleKind(ruleKind); + builder.setTargetLabel(targetLabel); + return builder.build(); + } + + /** + * Pre-processes an argument list, expanding options &at;filename to read in + * the content of the file and add it to the list of arguments. + * + * @param args the List of arguments to pre-process. + * @return the List of pre-processed arguments. + * @throws IOException if one of the files containing options cannot be read. + */ + private static Deque<String> expandArguments(List<String> args) throws IOException { + Deque<String> expanded = new ArrayDeque<>(args.size()); + for (String arg : args) { + expandArgument(expanded, arg); + } + return expanded; + } + + /** + * Expands a single argument, expanding options &at;filename to read in the content of the file + * and add it to the list of processed arguments. The &at; itself can be escaped with &at;&at;. + * + * @param arg the argument to pre-process. + * @return the list of pre-processed arguments. + * @throws IOException if one of the files containing options cannot be read. + */ + private static void expandArgument(Deque<String> expanded, String arg) + throws IOException { + if (arg.startsWith("@") && !arg.startsWith("@@")) { + for (String line : Files.readAllLines(Paths.get(arg.substring(1)), UTF_8)) { + if (line.length() > 0) { + expandArgument(expanded, line); + } + } + } else { + expanded.add(arg); + } + } + + /** + * Collects the arguments for a command line flag until it finds a flag that starts with the + * terminatorPrefix. + * + * @param output where to put the collected flag arguments. + * @param args + * @param terminatorPrefix the terminator prefix to stop collecting of argument flags. + */ + private static void collectFlagArguments( + ImmutableList.Builder<String> output, Deque<String> args, String terminatorPrefix) { + for (String arg = args.pollFirst(); arg != null; arg = args.pollFirst()) { + if (arg.startsWith(terminatorPrefix)) { + args.addFirst(arg); + break; + } + output.add(arg); + } + } + + /** + * Collects the arguments for the --processors command line flag until it finds a flag that + * starts with the terminatorPrefix. + * + * @param output where to put the collected flag arguments. + * @param args + * @param terminatorPrefix the terminator prefix to stop collecting of argument flags. + */ + private static void collectProcessorArguments( + List<String> output, Deque<String> args, String terminatorPrefix) + throws InvalidCommandLineException { + for (String arg = args.pollFirst(); arg != null; arg = args.pollFirst()) { + if (arg.startsWith(terminatorPrefix)) { + args.addFirst(arg); + break; + } + if (arg.contains(",")) { + throw new InvalidCommandLineException("processor argument may not contain commas: " + + arg); + } + output.add(arg); + } + } + + private static String getArgument(Deque<String> args, String arg) + throws InvalidCommandLineException { + try { + return args.remove(); + } catch (NoSuchElementException e) { + throw new InvalidCommandLineException(arg + ": missing argument"); + } + } + + private void processArg(String arg, Deque<String> args) + throws InvalidCommandLineException { + switch (arg) { + case "--sources": + collectFlagArguments(sourceFiles, args, "-"); + break; + case "--source_jars": + collectFlagArguments(sourceJars, args, "-"); + break; + case "--messages": + collectFlagArguments(messageFiles, args, "-"); + break; + case "--resources": + collectFlagArguments(resourceFiles, args, "-"); + break; + case "--resource_jars": + collectFlagArguments(resourceJars, args, "-"); + break; + case "--classpath_resources": + collectFlagArguments(rootResourceFiles, args, "-"); + break; + case "--classpath": + classPath = getArgument(args, arg); + break; + case "--processorpath": + processorPath = getArgument(args, arg); + break; + case "--processors": + collectProcessorArguments(processorNames, args, "-"); + break; + case "--output": + outputJar = getArgument(args, arg); + break; + case "--classdir": + classDir = getArgument(args, arg); + break; + case "--tempdir": + tempDir = getArgument(args, arg); + break; + case "--gendir": + // TODO(bazel-team) - remove when Bazel no longer passes this flag to buildjar. + getArgument(args, arg); + break; + case "--post_processor": + addExternalPostProcessor(postProcessors, args, arg); + break; + case "--compress_jar": + compressJar = true; + break; + case "--rule_kind": + ruleKind = getArgument(args, arg); + break; + case "--target_label": + targetLabel = getArgument(args, arg); + break; + default: + throw new InvalidCommandLineException("unknown option : '" + arg + "'"); + } + } + + private void addExternalPostProcessor(ImmutableList.Builder<AbstractPostProcessor> output, + Deque<String> args, String arg) throws InvalidCommandLineException { + String processorName = getArgument(args, arg); + ImmutableList.Builder<String> arguments = ImmutableList.builder(); + collectFlagArguments(arguments, args, "--"); + // TODO(bazel-team): there is no check than the same post processor is not added twice. + // We should either forbid multiple add of the same post processor or use a processor factory + // to allow multiple add of the same post processor. Anyway, this binary is invoked by Blaze + // and not manually. + output.add(AbstractPostProcessor.create(processorName, arguments.build())); + } + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/ReducedClasspathJavaLibraryBuilder.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/ReducedClasspathJavaLibraryBuilder.java new file mode 100644 index 0000000000..ebc1f0f006 --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/ReducedClasspathJavaLibraryBuilder.java @@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar; + +import com.google.devtools.build.buildjar.javac.JavacRunner; + +import com.sun.tools.javac.main.Main.Result; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; + +/** + * A variant of SimpleJavaLibraryBuilder that attempts to reduce the compile-time classpath right + * before invoking the compiler, based on extra information from provided .jdeps files. This mode is + * enabled via the --reduce_classpath flag, only when Blaze runs with --experimental_java_classpath. + * + * <p>A fall-back mechanism detects whether javac fails because the classpath is incorrectly + * discarding required entries, and re-attempts to compile with the full classpath. + */ +public class ReducedClasspathJavaLibraryBuilder extends SimpleJavaLibraryBuilder { + + /** + * Attempts to minimize the compile-time classpath before invoking javac, falling back to a + * regular compile. + * + * @param build A JavaLibraryBuildRequest request object describing what to compile + * @return result code of the javac compilation + * @throws IOException clean-up up the output directory fails + */ + @Override + Result compileSources(JavaLibraryBuildRequest build, JavacRunner javacRunner, PrintWriter err) + throws IOException { + // Minimize classpath, but only if we're actually compiling some sources (some invocations of + // JavaBuilder are only building resource jars). + String compressedClasspath = build.getClassPath(); + if (!build.getSourceFiles().isEmpty()) { + compressedClasspath = build.getDependencyModule() + .computeStrictClasspath(build.getClassPath(), build.getClassDir()); + } + String[] javacArguments = makeJavacArguments(build, compressedClasspath); + + // Compile! + StringWriter javacOutput = new StringWriter(); + PrintWriter javacOutputWriter = new PrintWriter(javacOutput); + Result result = javacRunner.invokeJavac(javacArguments, javacOutputWriter); + javacOutputWriter.close(); + + // If javac errored out because of missing entries on the classpath, give it another try. + // TODO(bazel-team): check performance impact of additional retries. + if (!result.isOK() && hasRecognizedError(javacOutput.toString())) { + if (debug) { + err.println("warning: [transitive] Target uses transitive classpath to compile."); + } + + // Reset output directories + prepareSourceCompilation(build); + + // Fall back to the regular compile, but add extra checks to catch transitive uses + javacArguments = makeJavacArguments(build); + result = javacRunner.invokeJavac(javacArguments, err); + } else { + err.print(javacOutput.getBuffer()); + } + return result; + } + + private boolean hasRecognizedError(String javacOutput) { + return javacOutput.contains("error: cannot access") + || javacOutput.contains("error: cannot find symbol") + || javacOutput.contains("com.sun.tools.javac.code.Symbol$CompletionFailure"); + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/SimpleJavaLibraryBuilder.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/SimpleJavaLibraryBuilder.java new file mode 100644 index 0000000000..714542f03e --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/SimpleJavaLibraryBuilder.java @@ -0,0 +1,137 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.buildjar.javac.JavacRunner; + +import com.sun.tools.javac.main.Main.Result; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +/** + * An implementation of the JavaBuilder that uses in-process javac to compile java files. + */ +class SimpleJavaLibraryBuilder extends AbstractJavaBuilder { + + @Override + Result compileSources(JavaLibraryBuildRequest build, JavacRunner javacRunner, PrintWriter err) + throws IOException { + String[] javacArguments = makeJavacArguments(build, build.getClassPath()); + return javacRunner.invokeJavac(javacArguments, err); + } + + @Override + protected void prepareSourceCompilation(JavaLibraryBuildRequest build) throws IOException { + super.prepareSourceCompilation(build); + + // Create sourceGenDir if necessary. + if (build.getSourceGenDir() != null) { + File sourceGenDir = new File(build.getSourceGenDir()); + if (sourceGenDir.exists()) { + try { + cleanupOutputDirectory(sourceGenDir); + } catch (IOException e) { + throw new IOException("Cannot clean output directory '" + sourceGenDir + "'", e); + } + } + sourceGenDir.mkdirs(); + } + } + + /** + * For the build configuration 'build', construct a command line that + * can be used for a javac invocation. + */ + protected String[] makeJavacArguments(JavaLibraryBuildRequest build) { + return makeJavacArguments(build, build.getClassPath()); + } + + /** + * For the build configuration 'build', construct a command line that + * can be used for a javac invocation. + */ + protected String[] makeJavacArguments(JavaLibraryBuildRequest build, String classPath) { + List<String> javacArguments = createInitialJavacArgs(build, classPath); + + javacArguments.addAll(getAnnotationProcessingOptions(build)); + + for (String option : build.getJavacOpts()) { + if (option.startsWith("-J")) { // ignore the VM options. + continue; + } + if (option.equals("-processor") || option.equals("-processorpath")) { + throw new IllegalStateException( + "Using " + option + " in javacopts is no longer supported." + + " Use a java_plugin() rule instead."); + } + javacArguments.add(option); + } + + javacArguments.addAll(build.getSourceFiles()); + return javacArguments.toArray(new String[0]); + } + + /** + * Given a JavaLibraryBuildRequest, computes the javac options for the annotation processing + * requested. + */ + private List<String> getAnnotationProcessingOptions(JavaLibraryBuildRequest build) { + List<String> args = new ArrayList<>(); + + // Javac treats "-processorpath ''" as setting the processor path to an empty list, + // whereas omitting the option is treated as not having a processor path (which causes + // processor path searches to fallback to the class path). + args.add("-processorpath"); + args.add( + build.getProcessorPath().isEmpty() ? "" : build.getProcessorPath()); + + if (!build.getProcessors().isEmpty() && !build.getSourceFiles().isEmpty()) { + // ImmutableSet.copyOf maintains order + ImmutableSet<String> deduplicatedProcessorNames = ImmutableSet.copyOf(build.getProcessors()); + args.add("-processor"); + args.add(Joiner.on(',').join(deduplicatedProcessorNames)); + + // Set javac output directory for generated sources. + if (build.getSourceGenDir() != null) { + args.add("-s"); + args.add(build.getSourceGenDir()); + } + } else { + // This is necessary because some jars contain discoverable annotation processors that + // previously didn't run, and they break builds if the "-proc:none" option is not passed to + // javac. + args.add("-proc:none"); + } + + return args; + } + + @Override + public void buildGensrcJar(JavaLibraryBuildRequest build, OutputStream err) + throws IOException { + JarCreator jar = new JarCreator(build.getGeneratedSourcesOutputJar()); + jar.setNormalize(true); + jar.setCompression(build.compressJar()); + jar.addDirectory(build.getSourceGenDir()); + jar.execute(); + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavaCompiler.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavaCompiler.java new file mode 100644 index 0000000000..9494ce7f49 --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavaCompiler.java @@ -0,0 +1,118 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar.javac; + +import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin; + +import com.sun.tools.javac.comp.AttrContext; +import com.sun.tools.javac.comp.CompileStates.CompileState; +import com.sun.tools.javac.comp.Env; +import com.sun.tools.javac.main.JavaCompiler; +import com.sun.tools.javac.util.Context; + +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; + +/** + * An extended version of the javac compiler, providing support for + * composable static analyses via a plugin mechanism. BlazeJavaCompiler + * keeps a list of plugins and calls callback methods in those plugins + * after certain compiler phases. The plugins perform the actual static + * analyses. + */ +public class BlazeJavaCompiler extends JavaCompiler { + + /** + * A list of plugins to run at particular points in the compile + */ + private final List<BlazeJavaCompilerPlugin> plugins = new ArrayList<>(); + + public BlazeJavaCompiler(Context context, Iterable<BlazeJavaCompilerPlugin> plugins) { + super(context); + + // initialize all plugins + for (BlazeJavaCompilerPlugin plugin : plugins) { + plugin.init(context, log, this); + this.plugins.add(plugin); + } + } + + /** + * Adds an initialization hook to the Context, such that each subsequent + * request for a JavaCompiler (i.e., a lookup for 'compilerKey' of our + * superclass, JavaCompiler) will actually construct and return our version + * of BlazeJavaCompiler. It's necessary since many new JavaCompilers may + * be requested for later stages of the compilation (annotation processing), + * within the same Context. And it's the preferred way for extending behavior + * within javac, per the documentation in {@link Context}. + */ + public static void preRegister(final Context context, + final Iterable<BlazeJavaCompilerPlugin> plugins) { + context.put(compilerKey, new Context.Factory<JavaCompiler>() { + @Override + public JavaCompiler make(Context c) { + return new BlazeJavaCompiler(c, plugins); + } + }); + } + + @Override + public Env<AttrContext> attribute(Env<AttrContext> env) { + Env<AttrContext> result = super.attribute(env); + + // Iterate over all plugins, calling their postAttribute methods + for (BlazeJavaCompilerPlugin plugin : plugins) { + plugin.postAttribute(result); + } + + return result; + } + + @Override + protected void flow(Env<AttrContext> env, Queue<Env<AttrContext>> results) { + if (compileStates.isDone(env, CompileState.FLOW)) { + super.flow(env, results); + return; + } + super.flow(env, results); + // Iterate over all plugins, calling their postFlow methods + for (BlazeJavaCompilerPlugin plugin : plugins) { + plugin.postFlow(env); + } + } + + @Override + public void close() { + for (BlazeJavaCompilerPlugin plugin : plugins) { + plugin.finish(); + } + plugins.clear(); + super.close(); + } + + /** + * Testing purposes only. Returns true if the collection of plugins in + * this instance contains one of the provided type. + */ + boolean pluginsContain(Class<? extends BlazeJavaCompilerPlugin> klass) { + for (BlazeJavaCompilerPlugin plugin : plugins) { + if (klass.isInstance(plugin)) { + return true; + } + } + return false; + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacLog.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacLog.java new file mode 100644 index 0000000000..6c0ee495fd --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacLog.java @@ -0,0 +1,88 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar.javac; + +import com.sun.tools.javac.util.Context; +import com.sun.tools.javac.util.JCDiagnostic; +import com.sun.tools.javac.util.Log; + +/** + * Log class for our custom patched javac. + * + * <p> This log class tweaks the standard javac log class so + * that it drops all non-errors after the first error that + * gets reported. By doing this, we + * ensure that all warnings are listed before all errors in javac's + * output. This makes life easier for everybody. + */ +public class BlazeJavacLog extends Log { + + private boolean hadError = false; + + /** + * Registers a custom BlazeJavacLog for the given context and -Werror spec. + * + * @param context Context + * @param warningsAsErrors Werror value + */ + public static void preRegister(final Context context) { + context.put(logKey, new Context.Factory<Log>() { + @Override + public Log make(Context c) { + return new BlazeJavacLog(c); + } + }); + } + + public static BlazeJavacLog instance(Context context) { + return (BlazeJavacLog) context.get(logKey); + } + + BlazeJavacLog(Context context) { + super(context); + } + + /** + * Returns true if we should display the note diagnostic + * passed in as argument, and false if we should discard + * it. + */ + private boolean shouldDisplayNote(JCDiagnostic diag) { + String noteCode = diag.getCode(); + return noteCode == null || + (!noteCode.startsWith("compiler.note.deprecated") && + !noteCode.startsWith("compiler.note.unchecked")); + } + + @Override + protected void writeDiagnostic(JCDiagnostic diag) { + switch (diag.getKind()) { + case NOTE: + if (shouldDisplayNote(diag)) { + super.writeDiagnostic(diag); + } + break; + case ERROR: + hadError = true; + super.writeDiagnostic(diag); + break; + default: + if (!hadError) { + // Do not print further warnings if an error has occured. + super.writeDiagnostic(diag); + } + } + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacMain.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacMain.java new file mode 100644 index 0000000000..92835880db --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacMain.java @@ -0,0 +1,202 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar.javac; + +import com.google.common.annotations.VisibleForTesting; +import com.google.devtools.build.buildjar.InvalidCommandLineException; +import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin; +import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin.PluginException; + +import com.sun.source.util.TaskEvent; +import com.sun.source.util.TaskListener; +import com.sun.tools.javac.api.JavacTaskImpl; +import com.sun.tools.javac.api.JavacTool; +import com.sun.tools.javac.api.MultiTaskListener; +import com.sun.tools.javac.main.Main; +import com.sun.tools.javac.main.Main.Result; +import com.sun.tools.javac.util.Context; +import com.sun.tools.javac.util.Options; +import com.sun.tools.javac.util.PropagatedException; + +import java.io.PrintWriter; +import java.util.Arrays; +import java.util.List; + +import javax.annotation.processing.Processor; +import javax.tools.DiagnosticListener; +import javax.tools.JavaFileManager; +import javax.tools.JavaFileObject; + +/** + * Main class for our custom patched javac. + * + * <p> This main class tweaks the standard javac log class by changing the + * compiler's context to use our custom log class. This custom log class + * modifies javac's output to list all errors after all warnings. + */ +public class BlazeJavacMain { + + /** + * Compose {@link com.sun.tools.javac.main.Main} and perform custom setup before deferring to + * its compile() method. + * + * Historically BlazeJavacMain extended javac's Main and overrode methods to get the desired + * custom behaviour. That approach created incompatibilities when upgrading to newer versions of + * javac, so composition is preferred. + */ + @VisibleForTesting + private List<BlazeJavaCompilerPlugin> plugins; + private final PrintWriter errOutput; + private final String compilerName; + + public BlazeJavacMain(PrintWriter errOutput, List<BlazeJavaCompilerPlugin> plugins) { + this.compilerName = "blaze javac"; + this.errOutput = errOutput; + this.plugins = plugins; + } + + /** + * Installs the BlazeJavaCompiler within the provided context. Enables + * plugins based on field values. + * + * @param context JavaCompiler's associated Context + */ + void setupBlazeJavaCompiler(Context context) { + preRegister(context, plugins); + } + + public void preRegister(Context context, List<BlazeJavaCompilerPlugin> plugins) { + this.plugins = plugins; + for (BlazeJavaCompilerPlugin plugin : plugins) { + plugin.initializeContext(context); + } + + BlazeJavacLog.preRegister(context); + BlazeJavaCompiler.preRegister(context, plugins); + } + + public Result compile(String[] argv) { + // set up a fresh Context with our custom bindings for JavaCompiler + Context context = new Context(); + // disable faulty Zip optimizations + Options options = Options.instance(context); + options.put("useOptimizedZip", "false"); + + // enable Java 8-style type inference features + // + // This is currently duplicated in JAVABUILDER. That's deliberate for now, because + // (1) JavaBuilder's integration test coverage for default options isn't very good, and + // (2) the migration from JAVABUILDER to java_toolchain configs is in progress so blaze + // integration tests for defaults options are also not trustworthy. + // + // TODO(bazel-team): removed duplication with JAVABUILDER + options.put("usePolyAttribution", "true"); + options.put("useStrictMethodClashCheck", "true"); + options.put("useStructuralMostSpecificResolution", "true"); + options.put("useGraphInference", "true"); + + String[] processedArgs; + + try { + processedArgs = processPluginArgs(argv); + } catch (InvalidCommandLineException e) { + errOutput.println(e.getMessage()); + return Result.CMDERR; + } + + setupBlazeJavaCompiler(context); + return compile(processedArgs, context); + } + + @VisibleForTesting + public Result compile(String[] argv, Context context) { + enableEndPositions(context); + try { + return new Main(compilerName, errOutput).compile(argv, context); + } catch (PropagatedException e) { + if (e.getCause() instanceof PluginException) { + PluginException pluginException = (PluginException) e.getCause(); + errOutput.println(pluginException.getMessage()); + return pluginException.getResult(); + } + e.printStackTrace(errOutput); + return Result.ABNORMAL; + } + } + + // javac9 removes the ability to pass lists of {@link JavaFileObject}s or {@link Processors}s to + // it's 'Main' class (i.e. the entry point for command-line javac). Having BlazeJavacMain + // continue to call into javac's Main has the nice property that it keeps JavaBuilder's + // behaviour closer to stock javac, but it makes it harder to write integration tests. This class + // provides a compile method that accepts file objects and processors, but it isn't + // guaranteed to behave exactly the same way as JavaBuilder does when used from the command-line. + // TODO(bazel-team): either stop using Main and commit to using the the API for everything, or + // re-write integration tests for JavaBuilder to use the real compile() method. + @VisibleForTesting + @Deprecated + public Result compile( + String[] argv, + Context context, + JavaFileManager fileManager, + DiagnosticListener<? super JavaFileObject> diagnosticListener, + List<JavaFileObject> javaFileObjects, + Iterable<? extends Processor> processors) { + + JavacTool tool = JavacTool.create(); + JavacTaskImpl task = (JavacTaskImpl) tool.getTask( + errOutput, + fileManager, + diagnosticListener, + Arrays.asList(argv), + null, + javaFileObjects, + context); + if (processors != null) { + task.setProcessors(processors); + } + + try { + return task.doCall(); + } catch (PluginException e) { + errOutput.println(e.getMessage()); + return e.getResult(); + } + } + + private static final TaskListener EMPTY_LISTENER = new TaskListener() { + @Override public void started(TaskEvent e) {} + @Override public void finished(TaskEvent e) {} + }; + + /** + * Convinces javac to run in 'API mode', and collect end position information needed by + * error-prone. + */ + private static void enableEndPositions(Context context) { + MultiTaskListener.instance(context).add(EMPTY_LISTENER); + } + + /** + * Processes Plugin-specific arguments and removes them from the args array. + */ + @VisibleForTesting + String[] processPluginArgs(String[] args) throws InvalidCommandLineException { + List<String> processedArgs = Arrays.asList(args); + for (BlazeJavaCompilerPlugin plugin : plugins) { + processedArgs = plugin.processArgs(processedArgs); + } + return processedArgs.toArray(new String[processedArgs.size()]); + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/JavacRunner.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/JavacRunner.java new file mode 100644 index 0000000000..6cd4b9abe3 --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/JavacRunner.java @@ -0,0 +1,55 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar.javac; + +import com.sun.tools.javac.main.Main.Result; + +import java.io.PrintWriter; + +/** + * The JavacRunner is a type that can be used to invoke + * javac and provides a convenient hook for modifications. + * It is split in two parts: An interface "JavacRunner" and + * an implementation of that interface, "JavacRunnerImpl". + * + * The type is split in two parts to allow us to load + * the implementation multiple times in different classloaders. + * This is neccessary, as a single javac can not run multiple + * times in parallel. By using different classloaders to load + * different copies of javac in different JavacRunnerImpls, + * we can run them in parallel. + * + * However, since each JavacRunnerImpl will then be loaded + * in a different classloader, we then would not be able to + * refer to it by simply declaring a type as "JavacRunnerImpl", + * as this refers to the JavacRunnerImpl type loaded with the + * default classloader. Therefore, we'd have to address each + * of the different JavacRunnerImpls as "Object" and invoke + * its method via reflection. + * + * We can circumvent this problem by declaring an interface + * that JavacRunnerImpl implements (i.e. JavacRunner). + * If we always load this super-interface in the default + * classloader, and make each JavacRunnerImpl (loaded in its + * own classloader) implement it, we can refer to the + * JavacRunnerImpls as "JavacRunner"s in the main program. + * That way, we can avoid using reflection and "Object" + * to deal with the different JavacRunnerImpls. + */ +public interface JavacRunner { + + Result invokeJavac(String[] args, PrintWriter output); + +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/JavacRunnerImpl.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/JavacRunnerImpl.java new file mode 100644 index 0000000000..a791ad6d03 --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/JavacRunnerImpl.java @@ -0,0 +1,47 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar.javac; + +import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin; + +import com.sun.tools.javac.main.Main.Result; + +import java.io.PrintWriter; +import java.util.List; + +/** + * This class wraps a single invocation of Javac. We + * invoke javac statically but wrap it with a synchronization. + * This is because the same javac cannot be invoked multiple + * times in parallel. + */ +public class JavacRunnerImpl implements JavacRunner { + + private final List<BlazeJavaCompilerPlugin> plugins; + + /** + * Passes extra information to BlazeJavacMain in case strict Java + * dependencies are enforced. + */ + public JavacRunnerImpl(List<BlazeJavaCompilerPlugin> plugins) { + this.plugins = plugins; + } + + @Override + public synchronized Result invokeJavac(String[] args, PrintWriter output) { + return new BlazeJavacMain(output, plugins).compile(args); + } + +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/BlazeJavaCompilerPlugin.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/BlazeJavaCompilerPlugin.java new file mode 100644 index 0000000000..e42713f222 --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/BlazeJavaCompilerPlugin.java @@ -0,0 +1,135 @@ +// Copyright 2011 Google Inc. All Rights Reserved. + +package com.google.devtools.build.buildjar.javac.plugins; + +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import com.google.devtools.build.buildjar.InvalidCommandLineException; + +import com.sun.tools.javac.comp.AttrContext; +import com.sun.tools.javac.comp.Env; +import com.sun.tools.javac.main.JavaCompiler; +import com.sun.tools.javac.main.Main.Result; +import com.sun.tools.javac.util.Context; +import com.sun.tools.javac.util.Log; +import com.sun.tools.javac.util.PropagatedException; + +import java.util.List; + +/** + * An interface for additional static analyses that need access to the javac compiler's AST at + * specific points in the compilation process. This class provides callbacks after the attribute and + * flow phases of the javac compilation process. A static analysis may be implemented by subclassing + * this abstract class and performing the analysis in the callback methods. The analysis may then be + * registered with the BlazeJavaCompiler to be run during the compilation process. See + * {@link com.google.devtools.build.buildjar.javac.plugins.dependency.StrictJavaDepsPlugin} for an + * example. + */ +public abstract class BlazeJavaCompilerPlugin { + + /** + * Allows plugins to pass errors through javac.Main to BlazeJavacMain and cleanly shut down the + * compiler. + */ + public static final class PluginException extends RuntimeException { + private final Result result; + private final String message; + + /** The compiler's exit status. */ + public Result getResult() { + return result; + } + + /** The message that will be printed to stderr before shutting down. */ + @Override + public String getMessage() { + return message; + } + + private PluginException(Result result, String message) { + this.result = result; + this.message = message; + } + } + + /** + * Pass an error through javac.Main to BlazeJavacMain and cleanly shut down the compiler. + */ + protected static Exception throwError(Result result, String message) { + // Javac re-throws exceptions wrapped by PropagatedException. + throw new PropagatedException(new PluginException(result, message)); + } + + protected Context context; + protected Log log; + protected JavaCompiler compiler; + + /** + * Preprocess the command-line flags that were passed to javac. This is called before + * {@link #init(Context, Log, JavaCompiler)} and {@link #initializeContext(Context)}. + * + * @param args The command-line flags that javac was invoked with. + * @throws InvalidCommandLineException if the arguments are invalid + * @returns The flags that do not belong to this plugin. + */ + public List<String> processArgs(List<String> args) throws InvalidCommandLineException { + return args; + } + + /** + * Called after all plugins have processed arguments and can be used to customize the Java + * compiler context. + */ + public void initializeContext(Context context) { + this.context = context; + } + + /** + * Performs analysis actions after the attribute phase of the javac compiler. + * The attribute phase performs symbol resolution on the parse tree. + * + * @param env The attributed parse tree (after symbol resolution) + */ + public void postAttribute(Env<AttrContext> env) {} + + /** + * Performs analysis actions after the flow phase of the javac compiler. + * The flow phase performs dataflow checks, such as finding unreachable + * statements. + * + * @param env The attributed parse tree (after symbol resolution) + */ + public void postFlow(Env<AttrContext> env) {} + + /** + * Performs analysis actions when the compiler is done and is about to wipe + * clean its internal data structures (such as the symbol table). + */ + public void finish() {} + + /** + * Initializes the plugin. Called by + * {@link com.google.devtools.build.buildjar.javac.BlazeJavaCompiler}'s constructor. + * + * @param context The Context object from the enclosing BlazeJavaCompiler instance + * @param log The Log object from the enclosing BlazeJavaCompiler instance + * @param compiler The enclosing BlazeJavaCompiler instance + */ + public void init(Context context, Log log, JavaCompiler compiler) { + this.context = context; + this.log = log; + this.compiler = compiler; + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/DependencyModule.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/DependencyModule.java new file mode 100644 index 0000000000..31964e5fd3 --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/DependencyModule.java @@ -0,0 +1,441 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar.javac.plugins.dependency; + +import com.google.common.annotations.VisibleForTesting; +import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin; +import com.google.devtools.build.lib.view.proto.Deps; +import com.google.devtools.build.lib.view.proto.Deps.Dependency.Kind; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Wrapper class for managing dependencies on top of + * {@link com.google.devtools.build.buildjar.javac.BlazeJavaCompiler}. If strict_java_deps is + * enabled, it keeps two maps between jar names (as they appear on the classpath) and their + * originating targets, one for direct dependencies and the other for transitive (indirect) + * dependencies, and enables the {@link StrictJavaDepsPlugin} to perform the actual checks. The + * plugin also collects dependency information during compilation, and DependencyModule generates a + * .jdeps artifact summarizing the discovered dependencies. + */ +public final class DependencyModule { + + public static enum StrictJavaDeps { + /** Legacy behavior: Silently allow referencing transitive dependencies. */ + OFF(false), + /** Warn about transitive dependencies being used directly. */ + WARN(true), + /** Fail the build when transitive dependencies are used directly. */ + ERROR(true); + + private boolean enabled; + + StrictJavaDeps(boolean enabled) { + this.enabled = enabled; + } + + /** Convenience method for just checking if it's not OFF */ + public boolean isEnabled() { + return enabled; + } + } + + private StrictJavaDeps strictJavaDeps = StrictJavaDeps.OFF; + private final Map<String, String> directJarsToTargets; + private final Map<String, String> indirectJarsToTargets; + private boolean strictClasspathMode; + private final Set<String> depsArtifacts; + private final String ruleKind; + private final String targetLabel; + private final String outputDepsFile; + private final String outputDepsProtoFile; + private final Set<String> usedClasspath; + private final Map<String, Deps.Dependency> explicitDependenciesMap; + private final Map<String, Deps.Dependency> implicitDependenciesMap; + Set<String> requiredClasspath; + + DependencyModule(StrictJavaDeps strictJavaDeps, + Map<String, String> directJarsToTargets, + Map<String, String> indirectJarsToTargets, + boolean strictClasspathMode, + Set<String> depsArtifacts, + String ruleKind, + String targetLabel, + String outputDepsFile, + String outputDepsProtoFile) { + this.strictJavaDeps = strictJavaDeps; + this.directJarsToTargets = directJarsToTargets; + this.indirectJarsToTargets = indirectJarsToTargets; + this.strictClasspathMode = strictClasspathMode; + this.depsArtifacts = depsArtifacts; + this.ruleKind = ruleKind; + this.targetLabel = targetLabel; + this.outputDepsFile = outputDepsFile; + this.outputDepsProtoFile = outputDepsProtoFile; + this.explicitDependenciesMap = new HashMap<>(); + this.implicitDependenciesMap = new HashMap<>(); + this.usedClasspath = new HashSet<>(); + } + + /** + * Returns a plugin to be enabled in the compiler. + */ + public BlazeJavaCompilerPlugin getPlugin() { + return new StrictJavaDepsPlugin(this); + } + + /** + * Writes the true, used compile-time classpath to the deps file, if specified. + */ + public void emitUsedClasspath(String classpath) throws IOException { + if (outputDepsFile != null) { + try (BufferedWriter out = new BufferedWriter(new FileWriter(outputDepsFile))) { + // Filter using the original classpath, to preserve ordering. + for (String entry : classpath.split(":")) { + if (usedClasspath.contains(entry)) { + out.write(entry); + out.newLine(); + } + } + } catch (IOException ex) { + throw new IOException("Cannot write dependencies to " + outputDepsFile, ex); + } + } + } + + /** + * Writes dependency information to the deps file in proto format, if specified. + * + * This is a replacement for {@link #emitUsedClasspath} above, which only outputs the used + * classpath. We collect more precise dependency information to allow Blaze to analyze both + * strict and unused dependencies based on the new deps.proto. + */ + public void emitDependencyInformation(String classpath, boolean successful) throws IOException { + if (outputDepsProtoFile == null) { + return; + } + + try (BufferedOutputStream out = + new BufferedOutputStream(new FileOutputStream(outputDepsProtoFile))) { + buildDependenciesProto(classpath, successful).writeTo(out); + } catch (IOException ex) { + throw new IOException("Cannot write dependencies to " + outputDepsProtoFile, ex); + } + } + + @VisibleForTesting + Deps.Dependencies buildDependenciesProto(String classpath, boolean successful) { + Deps.Dependencies.Builder deps = Deps.Dependencies.newBuilder(); + if (targetLabel != null) { + deps.setRuleLabel(targetLabel); + } + deps.setSuccess(successful); + // Filter using the original classpath, to preserve ordering. + for (String entry : classpath.split(":")) { + if (explicitDependenciesMap.containsKey(entry)) { + deps.addDependency(explicitDependenciesMap.get(entry)); + } else if (implicitDependenciesMap.containsKey(entry)) { + deps.addDependency(implicitDependenciesMap.get(entry)); + } + } + return deps.build(); + } + + /** + * Returns whether strict dependency checks (strictJavaDeps) are enabled. + */ + public boolean isStrictDepsEnabled() { + return strictJavaDeps.isEnabled(); + } + + /** + * Returns the mapping for jars of direct dependencies. The keys are full + * paths (as seen on the classpath), and the values are build target names. + */ + public Map<String, String> getDirectMapping() { + return directJarsToTargets; + } + + /** + * Returns the mapping for jars of indirect dependencies. The keys are full + * paths (as seen on the classpath), and the values are build target names. + */ + public Map<String, String> getIndirectMapping() { + return indirectJarsToTargets; + } + + /** + * Returns the strict dependency checking (strictJavaDeps) setting. + */ + public StrictJavaDeps getStrictJavaDeps() { + return strictJavaDeps; + } + + /** + * Returns the map collecting precise explicit dependency information. + */ + public Map<String, Deps.Dependency> getExplicitDependenciesMap() { + return explicitDependenciesMap; + } + + /** + * Returns the map collecting precise implicit dependency information. + */ + public Map<String, Deps.Dependency> getImplicitDependenciesMap() { + return implicitDependenciesMap; + } + + /** + * Returns the type (rule kind) of the originating target. + */ + public String getRuleKind() { + return ruleKind; + } + + /** + * Returns the name (label) of the originating target. + */ + public String getTargetLabel() { + return targetLabel; + } + + /** + * Returns the file name collecting dependency information. + */ + public String getOutputDepsFile() { + return outputDepsFile; + } + + @VisibleForTesting + Set<String> getUsedClasspath() { + return usedClasspath; + } + + /** + * Returns whether classpath reduction is enabled for this invocation. + */ + public boolean reduceClasspath() { + return strictClasspathMode; + } + + /** + * Computes a reduced compile-time classpath from the union of direct dependencies and their + * dependencies, as listed in the associated .deps artifacts. + */ + public String computeStrictClasspath(String originalClasspath, String classDir) { + if (!strictClasspathMode) { + return originalClasspath; + } + + // Classpath = direct deps + runtime direct deps + their .deps + requiredClasspath = new HashSet<>(directJarsToTargets.keySet()); + + for (String depsArtifact : depsArtifacts) { + collectDependenciesFromArtifact(depsArtifact); + } + + // Filter the initial classpath and keep the original order, with classDir as the last entry. + StringBuilder sb = new StringBuilder(); + String[] originalClasspathEntries = originalClasspath.split(":"); + + for (String entry : originalClasspathEntries) { + if (requiredClasspath.contains(entry)) { + sb.append(entry).append(":"); + } + } + sb.append(classDir); + return sb.toString(); + } + + @VisibleForTesting + void setStrictClasspath(Set<String> strictClasspath) { + this.requiredClasspath = strictClasspath; + } + + /** + * Updates {@link #requiredClasspath} to include dependencies from the given output artifact. + * + * During the .deps migration from text to proto format, this method will try to handle both. + * Blaze can thus switch the .deps artifacts independently. + */ + private void collectDependenciesFromArtifact(String path) { + // Try reading in proto format first + try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(path))) { + Deps.Dependencies deps = Deps.Dependencies.parseFrom(bis); + // Sanity check to make sure we have a valid proto, not a text file that happened to match. + if (!deps.hasRuleLabel()) { + throw new IOException("Text file"); + } + for (Deps.Dependency dep : deps.getDependencyList()) { + if (dep.getKind() == Kind.EXPLICIT || dep.getKind() == Kind.IMPLICIT) { + requiredClasspath.add(dep.getPath()); + } + } + } catch (IOException ex) { + // TODO(bazel-team): Remove this fallback to text format when Blaze is ready. + try (BufferedReader reader = new BufferedReader(new FileReader(path))) { + for (String dep = reader.readLine(); dep != null; dep = reader.readLine()) { + requiredClasspath.add(dep); + } + } catch (IOException exc) { + // At this point we can give up altogether + exc.printStackTrace(); + } + } + } + + /** + * Builder for {@link DependencyModule}. + */ + public static class Builder { + + private StrictJavaDeps strictJavaDeps = StrictJavaDeps.OFF; + private final Map<String, String> directJarsToTargets = new HashMap<>(); + private final Map<String, String> indirectJarsToTargets = new HashMap<>(); + private final Set<String> depsArtifacts = new HashSet<>(); + private String ruleKind; + private String targetLabel; + private String outputDepsFile; + private String outputDepsProtoFile; + private boolean strictClasspathMode = false; + + /** + * Constructs the DependencyModule, guaranteeing that the maps are + * never null (they may be empty), and the default strictJavaDeps setting + * is OFF. + * + * @return an instance of DependencyModule + */ + public DependencyModule build() { + return new DependencyModule(strictJavaDeps, directJarsToTargets, indirectJarsToTargets, + strictClasspathMode, depsArtifacts, ruleKind, targetLabel, outputDepsFile, + outputDepsProtoFile); + } + + /** + * Sets the strictness level for dependency checking. + * + * @param strictJavaDeps level, as specified by {@link StrictJavaDeps} + * @return this Builder instance + */ + public Builder setStrictJavaDeps(String strictJavaDeps) { + this.strictJavaDeps = StrictJavaDeps.valueOf(strictJavaDeps); + return this; + } + + /** + * Sets the type (rule kind) of the originating target. + * + * @param ruleKind kind, such as the rule kind of a RuleConfiguredTarget + * @return this Builder instance + */ + public Builder setRuleKind(String ruleKind) { + this.ruleKind = ruleKind; + return this; + } + + /** + * Sets the name (label) of the originating target. + * + * @param targetLabel label, such as the label of a RuleConfiguredTarget + * @return this Builder instance + */ + public Builder setTargetLabel(String targetLabel) { + this.targetLabel = targetLabel; + return this; + } + + /** + * Adds a direct mapping to the existing map for direct dependencies. + * + * @param jar path of jar artifact, as seen on classpath + * @param target full name of build target providing the jar + * @return this Builder instance + */ + public Builder addDirectMapping(String jar, String target) { + directJarsToTargets.put(jar, target); + return this; + } + + /** + * Adds an indirect mapping to the existing map for indirect dependencies. + * + * @param jar path of jar artifact, as seen on classpath + * @param target full name of build target providing the jar + * @return this Builder instance + */ + public Builder addIndirectMapping(String jar, String target) { + indirectJarsToTargets.put(jar, target); + return this; + } + + /** + * Sets the name of the file that will contain dependency information. + * + * @param outputDepsFile output file name for dependency information + * @return this Builder instance + */ + public Builder setOutputDepsFile(String outputDepsFile) { + this.outputDepsFile = outputDepsFile; + return this; + } + + /** + * Sets the name of the file that will contain dependency information in the protocol buffer + * format. + * + * @param outputDepsProtoFile output file name for dependency information + * @return this Builder instance + */ + public Builder setOutputDepsProtoFile(String outputDepsProtoFile) { + this.outputDepsProtoFile = outputDepsProtoFile; + return this; + } + + /** + * Adds a collection of dependency artifacts to use when reducing the compile-time classpath. + * + * @param depsArtifacts dependency artifacts + * @return this Builder instance + */ + public Builder addDepsArtifacts(Collection<String> depsArtifacts) { + this.depsArtifacts.addAll(depsArtifacts); + return this; + } + + /** + * Requests compile-time classpath reduction based on provided dependency artifacts. + * + * @return this Builder instance + */ + public Builder setReduceClasspath() { + this.strictClasspathMode = true; + return this; + } + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/ImplicitDependencyExtractor.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/ImplicitDependencyExtractor.java new file mode 100644 index 0000000000..df7190d787 --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/ImplicitDependencyExtractor.java @@ -0,0 +1,191 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar.javac.plugins.dependency; + +import com.google.devtools.build.lib.view.proto.Deps; + +import com.sun.tools.javac.code.Symbol.ClassSymbol; +import com.sun.tools.javac.code.Symtab; +import com.sun.tools.javac.file.ZipArchive; +import com.sun.tools.javac.file.ZipFileIndexArchive; +import com.sun.tools.javac.util.Context; + +import java.io.IOError; +import java.io.IOException; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import javax.lang.model.util.SimpleTypeVisitor7; +import javax.tools.JavaFileManager; +import javax.tools.JavaFileObject; +import javax.tools.StandardLocation; + +/** + * A lightweight mechanism for extracting compile-time dependencies from javac, by performing a scan + * of the symbol table after compilation finishes. It only includes dependencies from jar files, + * which can be interface jars or regular third_party jars, matching the compilation model of Blaze. + * Note that JDK8 may provide support for extracting per-class, finer-grained dependencies, and if + * that implementation has reasonable overhead it may be a future option. + */ +public class ImplicitDependencyExtractor { + + /** Set collecting dependencies names, used for the text output (soon to be removed) */ + private final Set<String> depsSet; + /** Map collecting dependency information, used for the proto output */ + private final Map<String, Deps.Dependency> depsMap; + private final TypeVisitor typeVisitor = new TypeVisitor(); + private final JavaFileManager fileManager; + + /** + * ImplicitDependencyExtractor does not guarantee any ordering of the reported + * dependencies. Clients should preserve the original classpath ordering + * if trying to minimize their classpaths using this information. + */ + public ImplicitDependencyExtractor(Set<String> depsSet, Map<String, Deps.Dependency> depsMap, + JavaFileManager fileManager) { + this.depsSet = depsSet; + this.depsMap = depsMap; + this.fileManager = fileManager; + } + + /** + * Collects the implicit dependencies of the given set of ClassSymbol roots. + * As we're interested in differentiating between symbols that were just + * resolved vs. symbols that were fully completed by the compiler, we start + * the analysis by finding all the implicit dependencies reachable from the + * given set of roots. For completeness, we then walk the symbol table + * associated with the given context and collect the jar files of the + * remaining class symbols found there. + * + * @param context compilation context + * @param roots root classes in the implicit dependency collection + */ + public void accumulate(Context context, Set<ClassSymbol> roots) { + Symtab symtab = Symtab.instance(context); + if (symtab.classes == null) { + return; + } + + // Collect transitive references for root types + for (ClassSymbol root : roots) { + root.type.accept(typeVisitor, null); + } + + Set<JavaFileObject> platformClasses = getPlatformClasses(fileManager); + + // Collect all other partially resolved types + for (ClassSymbol cs : symtab.classes.values()) { + if (cs.classfile != null) { + collectJarOf(cs.classfile, platformClasses); + } else if (cs.sourcefile != null) { + collectJarOf(cs.sourcefile, platformClasses); + } + } + } + + /** + * Collect the set of classes on the compilation bootclasspath. + * + * <p>TODO(bazel-team): this needs some work. JavaFileManager.list() is slower than + * StandardJavaFileManager.getLocation() and doesn't get cached. Additionally, tracking all + * classes in the bootclasspath requires a much bigger set than just tracking a list of jars. + * However, relying on the context containing a StandardJavaFileManager is brittle (e.g. Lombok + * wraps the file-manager in a ForwardingJavaFileManager.) + */ + public static HashSet<JavaFileObject> getPlatformClasses(JavaFileManager fileManager) { + HashSet<JavaFileObject> result = new HashSet<JavaFileObject>(); + Iterable<JavaFileObject> files; + try { + files = fileManager.list( + StandardLocation.PLATFORM_CLASS_PATH, "", EnumSet.of(JavaFileObject.Kind.CLASS), true); + } catch (IOException e) { + throw new IOError(e); + } + for (JavaFileObject file : files) { + result.add(file); + } + return result; + } + + /** + * Attempts to add the jar associated with the given JavaFileObject, if any, + * to the collection, filtering out jars on the compilation bootclasspath. + * + * @param reference JavaFileObject representing a class or source file + * @param platformClasses classes on javac's bootclasspath + */ + private void collectJarOf(JavaFileObject reference, Set<JavaFileObject> platformClasses) { + reference = unwrapFileObject(reference); + if (reference instanceof ZipArchive.ZipFileObject || + reference instanceof ZipFileIndexArchive.ZipFileIndexFileObject) { + // getName() will return something like com/foo/libfoo.jar(Bar.class) + String name = reference.getName().split("\\(")[0]; + // Filter out classes in rt.jar + if (!platformClasses.contains(reference)) { + depsSet.add(name); + if (!depsMap.containsKey(name)) { + depsMap.put(name, Deps.Dependency.newBuilder() + .setKind(Deps.Dependency.Kind.IMPLICIT) + .setPath(name) + .build()); + } + } + } + } + + + private static class TypeVisitor extends SimpleTypeVisitor7<Void, Void> { + // TODO(bazel-team): Override the visitor methods we're interested in. + } + + private static final Class<?> WRAPPED_JAVA_FILE_OBJECT = + getClassOrDie("com.sun.tools.javac.api.ClientCodeWrapper$WrappedJavaFileObject"); + + private static final java.lang.reflect.Field UNWRAP_FIELD = + getFieldOrDie( + getClassOrDie("com.sun.tools.javac.api.ClientCodeWrapper$WrappedFileObject"), + "clientFileObject"); + + private static Class<?> getClassOrDie(String name) { + try { + return Class.forName(name); + } catch (ClassNotFoundException e) { + throw new LinkageError(e.getMessage()); + } + } + + private static java.lang.reflect.Field getFieldOrDie(Class<?> clazz, String name) { + try { + java.lang.reflect.Field field = clazz.getDeclaredField(name); + field.setAccessible(true); + return field; + } catch (ReflectiveOperationException e) { + throw new LinkageError(e.getMessage()); + } + } + + public static JavaFileObject unwrapFileObject(JavaFileObject file) { + if (!file.getClass().equals(WRAPPED_JAVA_FILE_OBJECT)) { + return file; + } + try { + return (JavaFileObject) UNWRAP_FIELD.get(file); + } catch (ReflectiveOperationException e) { + throw new LinkageError(e.getMessage()); + } + } +} diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/StrictJavaDepsPlugin.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/StrictJavaDepsPlugin.java new file mode 100644 index 0000000000..1bdc73b621 --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/StrictJavaDepsPlugin.java @@ -0,0 +1,397 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.buildjar.javac.plugins.dependency; + +import static com.google.devtools.build.buildjar.javac.plugins.dependency.DependencyModule.StrictJavaDeps.ERROR; +import static com.google.devtools.build.buildjar.javac.plugins.dependency.ImplicitDependencyExtractor.getPlatformClasses; +import static com.google.devtools.build.buildjar.javac.plugins.dependency.ImplicitDependencyExtractor.unwrapFileObject; + +import com.google.common.annotations.VisibleForTesting; +import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin; +import com.google.devtools.build.lib.view.proto.Deps; +import com.google.devtools.build.lib.view.proto.Deps.Dependency; + +import com.sun.tools.javac.code.Flags; +import com.sun.tools.javac.code.Kinds; +import com.sun.tools.javac.code.Symbol; +import com.sun.tools.javac.code.Symbol.ClassSymbol; +import com.sun.tools.javac.comp.AttrContext; +import com.sun.tools.javac.comp.Env; +import com.sun.tools.javac.file.ZipArchive; +import com.sun.tools.javac.file.ZipFileIndexArchive; +import com.sun.tools.javac.main.JavaCompiler; +import com.sun.tools.javac.tree.JCTree; +import com.sun.tools.javac.tree.TreeScanner; +import com.sun.tools.javac.util.Context; +import com.sun.tools.javac.util.Log; +import com.sun.tools.javac.util.Log.WriterKind; + +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintWriter; +import java.text.MessageFormat; +import java.util.HashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.TreeSet; + +import javax.tools.JavaFileManager; +import javax.tools.JavaFileObject; + +/** + * A plugin for BlazeJavaCompiler that checks for types referenced directly + * in the source, but included through transitive dependencies. To get this + * information, we hook into the type attribution phase of the BlazeJavaCompiler + * (thus the overhead is another tree scan with the classic visitor). The + * constructor takes a map from jar names to target names, only for the jars that + * come from transitive dependencies (Blaze computes this information). + */ +public final class StrictJavaDepsPlugin extends BlazeJavaCompilerPlugin { + + @VisibleForTesting + static String targetMapping = + "com/google/devtools/build/buildjar/javac/resources/target.properties"; + + private static final String FIX_MESSAGE = + "%s** Command to add missing strict dependencies:%s\n" + + " add_dep %s%s\n\n"; + + private static final boolean USE_COLOR = true; + private ImplicitDependencyExtractor implicitDependencyExtractor; + private CheckingTreeScanner checkingTreeScanner; + private DependencyModule dependencyModule; + + /** Marks seen compilation toplevels and their import sections */ + private final Set<JCTree.JCCompilationUnit> toplevels; + /** Marks seen ASTs */ + private final Set<JCTree> trees; + + /** Computed missing dependencies */ + private final Set<String> missingTargets; + + private static Properties targetMap; + + private PrintWriter errWriter; + + /** + * On top of javac, we keep Blaze-specific information in the form of two + * maps. Both map jars (exactly as they appear on the classpath) to target + * names, one is used for direct dependencies, the other for the transitive + * dependencies. + * + * <p>This enables the detection of dependency issues. For instance, when a + * type com.Foo is referenced in the source and it's coming from an indirect + * dependency, we emit a warning flagging that dependency. Also, we can check + * whether the direct dependencies were actually necessary, i.e. if their + * associated jars were used at all for looking up class definitions. + */ + public StrictJavaDepsPlugin(DependencyModule dependencyModule) { + this.dependencyModule = dependencyModule; + toplevels = new HashSet<>(); + trees = new HashSet<>(); + targetMap = new Properties(); + missingTargets = new TreeSet<>(); + } + + @Override + public void init(Context context, Log log, JavaCompiler compiler) { + super.init(context, log, compiler); + errWriter = log.getWriter(WriterKind.ERROR); + JavaFileManager fileManager = context.get(JavaFileManager.class); + implicitDependencyExtractor = new ImplicitDependencyExtractor( + dependencyModule.getUsedClasspath(), dependencyModule.getImplicitDependenciesMap(), + fileManager); + checkingTreeScanner = context.get(CheckingTreeScanner.class); + if (checkingTreeScanner == null) { + Set<JavaFileObject> platformClasses = getPlatformClasses(fileManager); + checkingTreeScanner = new CheckingTreeScanner( + dependencyModule, log, missingTargets, platformClasses); + context.put(CheckingTreeScanner.class, checkingTreeScanner); + } + initTargetMap(); + } + + private void initTargetMap() { + try (InputStream is = getClass().getClassLoader().getResourceAsStream(targetMapping)) { + if (is != null) { + targetMap.load(is); + } + } catch (IOException ex) { + log.warning("Error loading Strict Java Deps mapping file: " + targetMapping, ex); + } + } + + /** + * We want to make another pass over the AST and "type-check" the usage + * of direct/transitive dependencies after the type attribution phase. + */ + @Override + public void postAttribute(Env<AttrContext> env) { + // We want to generate warnings/errors as if we were javac, and in order to + // use the internal log properly, we need to set its current source file + // information. The useSource call does just that, and is a common pattern + // from JavaCompiler: set source to current file and save the previous + // value, do work and generate warnings, reset source. + JavaFileObject prev = log.useSource( + env.enclClass.sym.sourcefile != null + ? env.enclClass.sym.sourcefile + : env.toplevel.sourcefile); + if (trees.add(env.tree)) { + checkingTreeScanner.scan(env.tree); + } + if (toplevels.add(env.toplevel)) { + checkingTreeScanner.scan(env.toplevel.getImports()); + } + log.useSource(prev); + } + + @Override + public void finish() { + implicitDependencyExtractor.accumulate(context, checkingTreeScanner.getSeenClasses()); + + if (!missingTargets.isEmpty()) { + StringBuilder missingTargetsStr = new StringBuilder(); + for (String target : missingTargets) { + missingTargetsStr.append(target); + missingTargetsStr.append(" "); + } + errWriter.print(String.format(FIX_MESSAGE, + USE_COLOR ? "\033[35m\033[1m" : "", + USE_COLOR ? "\033[0m" : "", + missingTargetsStr.toString(), + dependencyModule.getTargetLabel())); + } + } + + /** + * An AST visitor that implements our strict_java_deps checks. For now, it + * only emits warnings for types loaded from jar files provided by transitive + * (indirect) dependencies. Each type is considered only once, so at most one + * warning is generated for it. + */ + private static class CheckingTreeScanner extends TreeScanner { + + private static final String transitiveDepMessage = + "[strict] Using type {0} from an indirect dependency (TOOL_INFO: \"{1}\"). " + + "See command below **"; + + /** Lookup for jars coming from transitive dependencies */ + private final Map<String, String> indirectJarsToTargets; + + /** All error reporting is done through javac's log, */ + private final Log log; + + /** The strict_java_deps mode */ + private final DependencyModule.StrictJavaDeps strictJavaDepsMode; + + /** Missing targets */ + private final Set<String> missingTargets; + + /** Collect seen direct dependencies and their associated information */ + private final Map<String, Deps.Dependency> directDependenciesMap; + + /** We only emit one warning/error per class symbol */ + private final Set<ClassSymbol> seenClasses = new HashSet<>(); + private final Set<String> seenTargets = new HashSet<>(); + + /** The set of classes on the compilation bootclasspath. */ + private final Set<JavaFileObject> platformClasses; + + public CheckingTreeScanner(DependencyModule dependencyModule, Log log, + Set<String> missingTargets, Set<JavaFileObject> platformClasses) { + this.indirectJarsToTargets = dependencyModule.getIndirectMapping(); + this.strictJavaDepsMode = dependencyModule.getStrictJavaDeps(); + this.log = log; + this.missingTargets = missingTargets; + this.directDependenciesMap = dependencyModule.getExplicitDependenciesMap(); + this.platformClasses = platformClasses; + } + + Set<ClassSymbol> getSeenClasses() { + return seenClasses; + } + + /** + * Checks an AST node denoting a class type against direct/transitive + * dependencies. + */ + private void checkTypeLiteral(JCTree node) { + if (node == null || node.type.tsym == null) { + return; + } + + Symbol.TypeSymbol sym = node.type.tsym; + String jarName = getJarName(sym.enclClass(), platformClasses); + + // If this type symbol comes from a class file loaded from a jar, check + // whether that jar was a direct dependency and error out otherwise. + if (jarName != null && seenClasses.add(sym.enclClass())) { + collectExplicitDependency(jarName, node, sym); + } + } + + /** + * Marks the provided dependency as a direct/explicit dependency. Additionally, if + * strict_java_deps is enabled, it emits a [strict] compiler warning/error (behavior to be soon + * replaced by the more complete Blaze implementation). + */ + private void collectExplicitDependency(String jarName, JCTree node, Symbol.TypeSymbol sym) { + if (strictJavaDepsMode.isEnabled()) { + // Does it make sense to emit a warning/error for this pair of (type, target)? + // We want to emit only one error/warning per target. + String target = indirectJarsToTargets.get(jarName); + if (target != null && seenTargets.add(target)) { + String canonicalTargetName = canonicalizeTarget(target); + missingTargets.add(canonicalTargetName); + if (strictJavaDepsMode == ERROR) { + log.error(node.pos, "proc.messager", + MessageFormat.format(transitiveDepMessage, sym, canonicalTargetName)); + } else { + log.warning(node.pos, "proc.messager", + MessageFormat.format(transitiveDepMessage, sym, canonicalTargetName)); + } + } + } + + if (!directDependenciesMap.containsKey(jarName)) { + // Also update the dependency proto + Dependency dep = Dependency.newBuilder() + .setPath(jarName) + .setKind(Dependency.Kind.EXPLICIT) + .build(); + directDependenciesMap.put(jarName, dep); + } + } + + @Override + public void visitMethodDef(JCTree.JCMethodDecl method) { + if ((method.mods.flags & Flags.GENERATEDCONSTR) != 0) { + // If this is the constructor for an anonymous inner class, refrain from checking the + // compiler-generated method signature. Don't skip scanning the method body though, there + // might have been an anonymous initializer which still needs to be checked. + scan(method.body); + } else { + super.visitMethodDef(method); + } + } + + /** + * Visits an identifier in the AST. We only care about type symbols. + */ + @Override + public void visitIdent(JCTree.JCIdent tree) { + if (tree.sym != null && tree.sym.kind == Kinds.TYP) { + checkTypeLiteral(tree); + } + } + + /** + * Visits a field selection in the AST. We care because in some cases types + * may appear fully qualified and only inside a field selection + * (e.g., "com.foo.Bar.X", we want to catch the reference to Bar). + */ + @Override + public void visitSelect(JCTree.JCFieldAccess tree) { + scan(tree.selected); + if (tree.sym != null && tree.sym.kind == Kinds.TYP) { + checkTypeLiteral(tree); + } + } + + /** + * Visits an import statement. Static imports must not be omitted, as they + * are the only place we'll see the containing class references. + */ + @Override + public void visitImport(JCTree.JCImport tree) { + if (tree.isStatic()) { + scan(tree.getQualifiedIdentifier()); + } + } + } + + /** + * Returns the canonical version of the target name. Package private for testing. + */ + static String canonicalizeTarget(String target) { + String replacement = targetMap.getProperty(target); + if (replacement != null) { + return replacement; + } + int colonIndex = target.indexOf(':'); + if (colonIndex == -1) { + // No ':' in target, nothing to do. + return target; + } + int lastSlash = target.lastIndexOf('/', colonIndex); + if (lastSlash == -1) { + // No '/' or target is actually a filename in label format, return unmodified. + return target; + } + String packageName = target.substring(lastSlash + 1, colonIndex); + String suffix = target.substring(colonIndex + 1); + if (packageName.equals(suffix)) { + // target ends in "/something:something", canonicalize. + return target.substring(0, colonIndex); + } + return target; + } + + /** + * Returns the name of the jar file from which the given class symbol was + * loaded, if available, and null otherwise. Implicitly filters out jars + * from the compilation bootclasspath. + * @param platformClasses classes on javac's bootclasspath + */ + static String getJarName(ClassSymbol classSymbol, Set<JavaFileObject> platformClasses) { + if (classSymbol != null) { + // Ignore symbols that appear in the sourcepath: + if (haveSourceForSymbol(classSymbol)) { + return null; + } + JavaFileObject classfile = unwrapFileObject(classSymbol.classfile); + if (classfile instanceof ZipArchive.ZipFileObject + || classfile instanceof ZipFileIndexArchive.ZipFileIndexFileObject) { + String name = classfile.getName(); + // Here name will be something like blaze-out/.../com/foo/libfoo.jar(Bar.class) + String jarName = name.split("\\(")[0]; + if (!platformClasses.contains(classfile)) { + return jarName; + } + } + } + return null; + } + + /** + * Returns true if the given classSymbol corresponds to one of the sources being compiled. + */ + private static boolean haveSourceForSymbol(ClassSymbol classSymbol) { + if (classSymbol.sourcefile == null) { + return false; + } + + try { + // The classreader uses metadata to populate the symbol's sourcefile with a fake file object. + // Call getLastModified() to check if it's a real file: + classSymbol.sourcefile.getLastModified(); + } catch (UnsupportedOperationException e) { + return false; + } + + return true; + } +} |