diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java')
| -rw-r--r-- | src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java | 278 |
1 files changed, 278 insertions, 0 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java index df0c1977ab..cf68aeb6f4 100644 --- a/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java +++ b/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java @@ -13,13 +13,54 @@ // limitations under the License. package com.google.devtools.build.lib.syntax; +import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.events.Location; import com.google.devtools.build.lib.events.Location.LineAndColumn; import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.syntax.compiler.ByteCodeUtils; +import com.google.devtools.build.lib.syntax.compiler.DebugInfo; +import com.google.devtools.build.lib.syntax.compiler.LoopLabels; +import com.google.devtools.build.lib.syntax.compiler.ReflectionUtils; +import com.google.devtools.build.lib.syntax.compiler.VariableScope; import com.google.devtools.build.lib.vfs.PathFragment; +import net.bytebuddy.ByteBuddy; +import net.bytebuddy.asm.ClassVisitorWrapper; +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.description.modifier.MethodManifestation; +import net.bytebuddy.description.modifier.Ownership; +import net.bytebuddy.description.modifier.TypeManifestation; +import net.bytebuddy.description.modifier.Visibility; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.DynamicType.Unloaded; +import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; +import net.bytebuddy.implementation.Implementation; +import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.implementation.bytecode.ByteCodeAppender; +import net.bytebuddy.implementation.bytecode.member.MethodReturn; +import net.bytebuddy.matcher.ElementMatchers; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Label; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.util.Textifier; +import org.objectweb.asm.util.TraceClassVisitor; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + /** * The actual function registered in the environment. This function is defined in the * parsed code using {@link FunctionDefStatement}. @@ -31,12 +72,20 @@ public class UserDefinedFunction extends BaseFunction { // we close over the globals at the time of definition private final Environment.Frame definitionGlobals; + private Optional<Method> method; + // TODO(bazel-team) make this configurable once the compiler is stable + public static boolean debugCompiler = false; + public static boolean debugCompilerPrintByteCode = false; + private static File debugFolder; + public static boolean enableCompiler = false; + protected UserDefinedFunction(Identifier function, FunctionSignature.WithValues<Object, SkylarkType> signature, ImmutableList<Statement> statements, Environment.Frame definitionGlobals) { super(function.getName(), signature, function.getLocation()); this.statements = statements; this.definitionGlobals = definitionGlobals; + method = enableCompiler ? buildCompiledFunction() : Optional.<Method>absent(); } public FunctionSignature.WithValues<Object, SkylarkType> getFunctionSignature() { @@ -59,6 +108,13 @@ public class UserDefinedFunction extends BaseFunction { getName(), Iterables.getLast(env.getStackTrace()).getName())); } + if (enableCompiler && method.isPresent()) { + Object returnValue = callCompiledFunction(arguments, ast, env); + if (returnValue != null) { + return returnValue; + } + } + Profiler.instance().startTask(ProfilerTask.SKYLARK_USER_FN, getLocationPathAndLine() + "#" + getName()); try { @@ -91,6 +147,161 @@ public class UserDefinedFunction extends BaseFunction { } } + private Object callCompiledFunction(Object[] arguments, FuncallExpression ast, Environment env) { + compilerDebug("Calling compiled function " + getLocationPathAndLine() + " " + getName()); + try { + env.enterScope(this, ast, definitionGlobals); + + return method + .get() + .invoke(null, ImmutableList.builder().add(arguments).add(env).build().toArray()); + + } catch (IllegalAccessException e) { + // this should never happen + throw new RuntimeException( + "Compiler created code that could not be accessed reflectively.", e); + } catch (InvocationTargetException e) { + compilerDebug("Error running compiled version", e.getCause()); + return null; + } finally { + env.exitScope(); + } + } + + /** + * Generates a subclass of {@link CompiledFunction} with a static method "call" and static + * methods for getting information from a {@link DebugInfo} instance. + * + * <p>The "call" method contains the compiled version of this function's AST. + */ + private Optional<Method> buildCompiledFunction() { + // replace the / character in the path so we have file system compatible class names + // the java specification mentions that $ should be used in generated code + // see http://docs.oracle.com/javase/specs/jls/se7/html/jls-3.html#jls-3.8 + String path = + location.getPath() != null ? location.getPath().getPathString().replace('/', '$') : ""; + String compiledFunctionClassName = + CompiledFunction.class.getCanonicalName() + path + "$" + getName(); + compilerDebug("Compiling " + getLocationPathAndLine() + " " + getName()); + try { + int publicStatic = Visibility.PUBLIC.getMask() | Ownership.STATIC.getMask(); + TypeDescription.Latent latentCompiledFunctionClass = + new TypeDescription.Latent( + compiledFunctionClassName, + publicStatic | TypeManifestation.FINAL.getMask(), + new TypeDescription.ForLoadedType(CompiledFunction.class), + Collections.<TypeDescription>emptyList()); + MethodDescription getAstNode = + new MethodDescription.Latent( + latentCompiledFunctionClass, + new MethodDescription.Token( + "getAstNode", + publicStatic | MethodManifestation.FINAL.getMask(), + new TypeDescription.ForLoadedType(ASTNode.class), + Arrays.asList(new TypeDescription.ForLoadedType(int.class)))); + MethodDescription getLocation = + new MethodDescription.Latent( + latentCompiledFunctionClass, + new MethodDescription.Token( + "getLocation", + publicStatic | MethodManifestation.FINAL.getMask(), + new TypeDescription.ForLoadedType(Location.class), + Arrays.asList(new TypeDescription.ForLoadedType(int.class)))); + + DebugInfo debugInfo = new DebugInfo(getAstNode, getLocation); + FunctionSignature sig = signature.getSignature(); + VariableScope scope = VariableScope.function(sig.getNames()); + Implementation compiledImplementation = compileBody(scope, debugInfo); + + List<Class<?>> parameterTypes = sig.getShape().toClasses(); + parameterTypes.add(Environment.class); + Unloaded<CompiledFunction> unloadedImplementation = + new ByteBuddy() + .withClassVisitor(new StackMapFrameClassVisitor(debugCompilerPrintByteCode)) + .subclass(CompiledFunction.class) + .name(compiledFunctionClassName) + .defineMethod( + "call", + Object.class, + parameterTypes, + Visibility.PUBLIC, + Ownership.STATIC, + MethodManifestation.FINAL) + .intercept(compiledImplementation) + .defineMethod(getAstNode) + // TODO(bazel-team) unify the two delegate fields into one, probably needs a custom + // ImplementationDelegate that adds it only once? or just create the static field + // itself with the correct value and create getAstNode & getLocation with a custom + // implementation using it + .intercept( + MethodDelegation.to(debugInfo, DebugInfo.class, "getAstNodeDelegate") + .filter(ElementMatchers.named("getAstNode"))) + .defineMethod(getLocation) + .intercept( + MethodDelegation.to(debugInfo, DebugInfo.class, "getLocationDelegate") + .filter(ElementMatchers.named("getLocation"))) + .make(); + saveByteCode(unloadedImplementation); + Class<? extends CompiledFunction> functionClass = + unloadedImplementation + .load(getClass().getClassLoader(), ClassLoadingStrategy.Default.WRAPPER) + .getLoaded(); + + return Optional.of( + ReflectionUtils.getMethod( + functionClass, + "call", + parameterTypes.toArray(new Class<?>[parameterTypes.size()])) + .getLoadedMethod()); + } catch (Throwable e) { + compilerDebug("Error while compiling", e); + // TODO(bazel-team) don't capture all throwables? couldn't compile this, log somewhere? + return Optional.absent(); + } + } + + /** + * Saves byte code to a temporary directory prefixed with "skylarkbytecode" in the system + * default temporary directory. + */ + private void saveByteCode(Unloaded<CompiledFunction> unloadedImplementation) { + if (debugCompiler) { + try { + if (debugFolder == null) { + debugFolder = Files.createTempDirectory("skylarkbytecode").toFile(); + } + unloadedImplementation.saveIn(debugFolder); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + /** + * Builds a byte code implementation of the AST. + */ + private Implementation compileBody(VariableScope scope, DebugInfo debugInfo) { + List<ByteCodeAppender> code = new ArrayList<>(statements.size()); + code.add(null); // reserve space for later addition of the local variable initializer + + for (Statement statement : statements) { + code.add(statement.compile(scope, LoopLabels.ABSENT, debugInfo)); + } + // add a return None if there are no statements or the last one to ensure the method always + // returns something. This implements the interpreters behavior. + if (statements.isEmpty() + || !(statements.get(statements.size() - 1) instanceof ReturnStatement)) { + code.add(new ByteCodeAppender.Simple(Runtime.GET_NONE, MethodReturn.REFERENCE)); + } + // we now know which variables we used in the method, so assign them "undefined" (i.e. null) + // at the beginning of the method + code.set(0, scope.createLocalVariablesUndefined()); + // TODO(bazel-team) wrap ByteCodeAppender in our own type including a reference to the ASTNode + // it came from and verify the stack and local variables ourselves, because ASM does not help + // with debugging much when its stack map frame calculation fails because of invalid byte code + return new Implementation.Simple(ByteCodeUtils.compoundAppender(code)); + } + /** * Returns the location (filename:line) of the BaseFunction's definition. * @@ -113,4 +324,71 @@ public class UserDefinedFunction extends BaseFunction { } return builder.toString(); } + + private void compilerDebug(String message) { + System.err.println(message); + } + + private void compilerDebug(String message, Throwable e) { + compilerDebug(message); + e.printStackTrace(); + } + + /** + * A simple super class for all compiled function's classes. + */ + protected abstract static class CompiledFunction {} + + /** + * A {@link Textifier} for printing the generated byte code that keeps the ASM-internal label + * names in place for easier debugging with IDE debuggers. + */ + private static class DebugTextifier extends Textifier { + DebugTextifier() { + super(Opcodes.ASM5); + } + + @Override + protected void appendLabel(Label l) { + buf.append(l.toString()); + } + + @Override + protected Textifier createTextifier() { + return new DebugTextifier(); + } + } + + /** + * Passes the {@link ClassWriter#COMPUTE_FRAMES} hint to ASM and optionally prints generated + * byte code to System.err. + */ + private static class StackMapFrameClassVisitor implements ClassVisitorWrapper { + + private final boolean debug; + + private StackMapFrameClassVisitor(boolean debug) { + this.debug = debug; + } + + @Override + public int mergeWriter(int hint) { + return hint | ClassWriter.COMPUTE_FRAMES; + } + + @Override + public int mergeReader(int hint) { + return hint; + } + + @Override + public ClassVisitor wrap(ClassVisitor classVisitor) { + if (debug) { + return new TraceClassVisitor( + classVisitor, new DebugTextifier(), new PrintWriter(System.err, true)); + } else { + return classVisitor; + } + } + } } |