From 768cbc473fa4c27aa5fef1563b2a7c0c13ef10dd Mon Sep 17 00:00:00 2001 From: Googler Date: Fri, 28 Aug 2015 12:52:14 +0000 Subject: Add profiling for Skylark lexer, parser, user- and built-in functions. -- MOS_MIGRATED_REVID=101769963 --- .../google/devtools/build/lib/profiler/ProfilerTask.java | 4 ++++ .../google/devtools/build/lib/syntax/BuiltinFunction.java | 7 +++++-- .../java/com/google/devtools/build/lib/syntax/Lexer.java | 4 ++++ .../java/com/google/devtools/build/lib/syntax/Parser.java | 14 +++++++++----- .../devtools/build/lib/syntax/UserDefinedFunction.java | 5 +++++ 5 files changed, 27 insertions(+), 7 deletions(-) (limited to 'src/main/java/com/google/devtools/build/lib') diff --git a/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java index 5537bf0bff..37349b1406 100644 --- a/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java +++ b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java @@ -81,6 +81,10 @@ public enum ProfilerTask { CONSTRUCT_INCLUDE_PATHS("construct include paths"), PARSE_AND_HINTS_RESULTS("parse and hints results"), PROCESS_RESULTS_AND_ENQUEUE("process results and enqueue"), + SKYLARK_LEXER("Skylark Lexer"), + SKYLARK_PARSER("Skylark Parser"), + SKYLARK_USER_FN("Skylark user function call", -1, 0xCC0033, 0), + SKYLARK_BUILTIN_FN("Skylark builtin function call", -1, 0x990033, 0), UNKNOWN("Unknown event", -1, 0x339966, 0); // Size of the ProfilerTask value space. diff --git a/src/main/java/com/google/devtools/build/lib/syntax/BuiltinFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/BuiltinFunction.java index 6b94d325bc..9fcf9a4403 100644 --- a/src/main/java/com/google/devtools/build/lib/syntax/BuiltinFunction.java +++ b/src/main/java/com/google/devtools/build/lib/syntax/BuiltinFunction.java @@ -15,16 +15,16 @@ package com.google.devtools.build.lib.syntax; import com.google.common.base.Preconditions; import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.syntax.SkylarkSignatureProcessor.HackHackEitherList; import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType; - import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.NoSuchElementException; import java.util.concurrent.ExecutionException; - import javax.annotation.Nullable; /** @@ -145,6 +145,7 @@ public class BuiltinFunction extends BaseFunction { } } + long startTime = Profiler.nanoTimeMaybe(); // Last but not least, actually make an inner call to the function with the resolved arguments. try { return invokeMethod.invoke(this, args); @@ -185,6 +186,8 @@ public class BuiltinFunction extends BaseFunction { throw badCallException(loc, e, args); } catch (IllegalAccessException e) { throw badCallException(loc, e, args); + } finally { + Profiler.instance().logSimpleTask(startTime, ProfilerTask.SKYLARK_BUILTIN_FN, getName()); } } diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java index 9d148861bf..6a7a14a8c1 100644 --- a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java +++ b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java @@ -19,6 +19,8 @@ import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.util.Pair; import com.google.devtools.build.lib.vfs.PathFragment; @@ -98,7 +100,9 @@ public final class Lexer { this.locationInfo = new LocationInfo(input.getPath(), lineNumberTable); indentStack.push(0); + long startTime = Profiler.nanoTimeMaybe(); tokenize(); + Profiler.instance().logSimpleTask(startTime, ProfilerTask.SKYLARK_LEXER, getFilename()); } public Lexer(ParserInputSource input, EventHandler eventHandler) { diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Parser.java b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java index 6334673aa1..bbed96e332 100644 --- a/src/main/java/com/google/devtools/build/lib/syntax/Parser.java +++ b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java @@ -27,6 +27,8 @@ import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.events.Location; import com.google.devtools.build.lib.packages.CachingPackageLocator; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.syntax.DictionaryLiteral.DictionaryEntryLiteral; import com.google.devtools.build.lib.syntax.IfStatement.ConditionalStatements; import com.google.devtools.build.lib.vfs.Path; @@ -339,7 +341,7 @@ class Parser { // Keywords that exist in Python and that we don't parse. private static final EnumSet FORBIDDEN_KEYWORDS = - EnumSet.of(TokenKind.AS, TokenKind.ASSERT, + EnumSet.of(TokenKind.AS, TokenKind.ASSERT, TokenKind.DEL, TokenKind.EXCEPT, TokenKind.FINALLY, TokenKind.FROM, TokenKind.GLOBAL, TokenKind.IMPORT, TokenKind.IS, TokenKind.LAMBDA, TokenKind.NONLOCAL, TokenKind.RAISE, TokenKind.TRY, TokenKind.WITH, TokenKind.WHILE, TokenKind.YIELD); @@ -1056,6 +1058,7 @@ class Parser { // file_input ::= ('\n' | stmt)* EOF private List parseFileInput() { + long startTime = Profiler.nanoTimeMaybe(); List list = new ArrayList<>(); while (token.kind != TokenKind.EOF) { if (token.kind == TokenKind.NEWLINE) { @@ -1069,6 +1072,7 @@ class Parser { parseTopLevelStatement(list); } } + Profiler.instance().logSimpleTask(startTime, ProfilerTask.SKYLARK_PARSER, includedFiles); return list; } @@ -1079,10 +1083,10 @@ class Parser { expect(TokenKind.STRING); return; } - + Token pathToken = token; String path = (String) token.value; - + nextToken(); expect(TokenKind.COMMA); @@ -1098,7 +1102,7 @@ class Parser { parseLoadSymbol(symbols); } expect(TokenKind.RPAREN); - + LoadStatement stmt = new LoadStatement(path, symbols); // Although validateLoadPath() is invoked as part of validate(ValidationEnvironment), @@ -1490,7 +1494,7 @@ class Parser { int start = token.left; int end = token.right; expect(TokenKind.RETURN); - + Expression expression; if (STATEMENT_TERMINATOR_SET.contains(token.kind)) { // this None makes the AST not correspond to the source exactly anymore diff --git a/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java index 750429d65b..f9719ae15d 100644 --- a/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java +++ b/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java @@ -15,6 +15,8 @@ package com.google.devtools.build.lib.syntax; import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; /** * The actual function registered in the environment. This function is defined in the @@ -57,6 +59,7 @@ public class UserDefinedFunction extends BaseFunction { env.update(name, arguments[i++]); } + long startTimeProfiler = Profiler.nanoTimeMaybe(); Statement lastStatement = null; try { for (Statement stmt : statements) { @@ -74,6 +77,8 @@ public class UserDefinedFunction extends BaseFunction { new EvalExceptionWithStackTrace(ex, lastStatement.getLocation()); real.registerStatement(lastStatement); throw real; + } finally { + Profiler.instance().logSimpleTask(startTimeProfiler, ProfilerTask.SKYLARK_USER_FN, getName()); } return Environment.NONE; } -- cgit v1.2.3