From cfc64a3910ed7c144763f4156a9b4979c5f15d96 Mon Sep 17 00:00:00 2001 From: shahan Date: Fri, 11 May 2018 09:46:54 -0700 Subject: Records profiling information for ActionFS staging and updates. PiperOrigin-RevId: 196266567 --- scripts/BUILD | 2 +- src/BUILD | 2 +- .../java/com/google/testing/coverage/BUILD | 2 +- .../devtools/build/lib/profiler/ProfilerTask.java | 2 + .../build/lib/skyframe/ActionFileSystem.java | 84 ++++++++++++---------- .../google/devtools/build/android/desugar/BUILD | 2 +- 6 files changed, 54 insertions(+), 40 deletions(-) diff --git a/scripts/BUILD b/scripts/BUILD index ee6ef86c37..32f2a39c31 100644 --- a/scripts/BUILD +++ b/scripts/BUILD @@ -18,7 +18,7 @@ genrule( output_to_bindir = 1, # Bazel 0.8.0 doesn't have this target under @bazel_tools, so we have to # use it from the main repository - toolchains = ["@bazel_tools//tools/jdk:current_java_runtime"], + toolchains = ["@bazel_tools//tools/jdk:current_host_java_runtime"], tools = [ ":generate_bash_completion.sh", "//src:bazel", diff --git a/src/BUILD b/src/BUILD index 8f5da4f0d4..9c30902162 100644 --- a/src/BUILD +++ b/src/BUILD @@ -361,7 +361,7 @@ genrule( # Technically, this should be prefixed with @bazel_tools, but in order to # avoid having to wait a Bazel release which has this target under # @bazel_tools, we use it from the main repository. It's the same anyway. - toolchains = ["@bazel_tools//tools/jdk:current_java_runtime"], + toolchains = ["@bazel_tools//tools/jdk:current_host_java_runtime"], tools = ["combine_derived_java_srcs.sh"], visibility = ["//:__pkg__"], ) diff --git a/src/java_tools/junitrunner/java/com/google/testing/coverage/BUILD b/src/java_tools/junitrunner/java/com/google/testing/coverage/BUILD index 031793165f..a5055017e0 100644 --- a/src/java_tools/junitrunner/java/com/google/testing/coverage/BUILD +++ b/src/java_tools/junitrunner/java/com/google/testing/coverage/BUILD @@ -70,7 +70,7 @@ genrule( "rm -fr \"$${JARJAR}\"", ]), tags = ["manual"], - toolchains = ["@bazel_tools//tools/jdk:current_java_runtime"], + toolchains = ["@bazel_tools//tools/jdk:current_host_java_runtime"], tools = [ "//src/java_tools/singlejar:SingleJar_deploy.jar", "//third_party/jarjar:jarjar_bin_deploy.jar", diff --git a/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java index 10560db143..804a154721 100644 --- a/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java +++ b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java @@ -89,6 +89,8 @@ public enum ProfilerTask { SKYLARK_USER_FN("Skylark user function call", -1, 0xCC0033, 0), SKYLARK_BUILTIN_FN("Skylark builtin function call", -1, 0x990033, 0), SKYLARK_USER_COMPILED_FN("Skylark compiled user function call", -1, 0xCC0033, 0), + ACTION_FS_STAGING("Staging per-action file system", -1, 0x000000, 0), + ACTION_FS_UPDATE("Updating per-action file system", -1, 0x000000, 0), UNKNOWN("Unknown event", -1, 0x339966, 0); // Size of the ProfilerTask value space. diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionFileSystem.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionFileSystem.java index 62e9761abd..6ebc0cbcf2 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ActionFileSystem.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionFileSystem.java @@ -23,6 +23,8 @@ import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionInputFileCache; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.FileStateType; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.vfs.AbstractFileSystem; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; @@ -89,33 +91,38 @@ final class ActionFileSystem extends AbstractFileSystem implements ActionInputFi Map inputData, Iterable allowedInputs, Iterable outputArtifacts) { - roots.add(computeExecRoot(outputArtifacts)); - - // TODO(shahan): Underestimates because this doesn't account for discovered inputs. Improve - // this estimate using data. - this.reverseMap = new ConcurrentHashMap<>(inputData.size()); - - HashMap inputs = new HashMap<>(); - for (Map.Entry entry : inputData.entrySet()) { - Artifact input = entry.getKey(); - updateRootsIfSource(input); - inputs.put(input.getExecPath(), new SimpleArtifactAndMetadata(input, entry.getValue())); - updateReverseMapIfDigestExists(entry.getValue(), entry.getKey()); - } - for (Artifact input : allowedInputs) { - PathFragment execPath = input.getExecPath(); - inputs.computeIfAbsent(execPath, unused -> new OptionalInputArtifactAndMetadata(input)); - updateRootsIfSource(input); - } - this.inputs = inputs; + try { + Profiler.instance().startTask(ProfilerTask.ACTION_FS_STAGING, "staging"); + roots.add(computeExecRoot(outputArtifacts)); + + // TODO(shahan): Underestimates because this doesn't account for discovered inputs. Improve + // this estimate using data. + this.reverseMap = new ConcurrentHashMap<>(inputData.size()); + + HashMap inputs = new HashMap<>(); + for (Map.Entry entry : inputData.entrySet()) { + Artifact input = entry.getKey(); + updateRootsIfSource(input); + inputs.put(input.getExecPath(), new SimpleArtifactAndMetadata(input, entry.getValue())); + updateReverseMapIfDigestExists(entry.getValue(), entry.getKey()); + } + for (Artifact input : allowedInputs) { + PathFragment execPath = input.getExecPath(); + inputs.computeIfAbsent(execPath, unused -> new OptionalInputArtifactAndMetadata(input)); + updateRootsIfSource(input); + } + this.inputs = inputs; - validateRoots(); + validateRoots(); - this.outputs = - Streams.stream(outputArtifacts) - .collect( - ImmutableMap.toImmutableMap( - a -> a.getExecPath(), a -> new ArtifactAndMutableMetadata(a))); + this.outputs = + Streams.stream(outputArtifacts) + .collect( + ImmutableMap.toImmutableMap( + a -> a.getExecPath(), a -> new ArtifactAndMutableMetadata(a))); + } finally { + Profiler.instance().completeTask(ProfilerTask.ACTION_FS_STAGING); + } } /** @@ -132,18 +139,23 @@ final class ActionFileSystem extends AbstractFileSystem implements ActionInputFi /** Input discovery changes the values of the input data map so it must be updated accordingly. */ public void updateInputData(Map inputData) { - boolean foundNewRoots = false; - for (Map.Entry entry : inputData.entrySet()) { - ArtifactAndMetadata current = inputs.get(entry.getKey().getExecPath()); - if (current == null || isUnsetOptional(current)) { - Artifact input = entry.getKey(); - inputs.put(input.getExecPath(), new SimpleArtifactAndMetadata(input, entry.getValue())); - foundNewRoots = updateRootsIfSource(entry.getKey()) || foundNewRoots; - updateReverseMapIfDigestExists(entry.getValue(), entry.getKey()); + try { + Profiler.instance().startTask(ProfilerTask.ACTION_FS_UPDATE, "update"); + boolean foundNewRoots = false; + for (Map.Entry entry : inputData.entrySet()) { + ArtifactAndMetadata current = inputs.get(entry.getKey().getExecPath()); + if (current == null || isUnsetOptional(current)) { + Artifact input = entry.getKey(); + inputs.put(input.getExecPath(), new SimpleArtifactAndMetadata(input, entry.getValue())); + foundNewRoots = updateRootsIfSource(entry.getKey()) || foundNewRoots; + updateReverseMapIfDigestExists(entry.getValue(), entry.getKey()); + } } - } - if (foundNewRoots) { - validateRoots(); + if (foundNewRoots) { + validateRoots(); + } + } finally { + Profiler.instance().completeTask(ProfilerTask.ACTION_FS_UPDATE); } } diff --git a/src/test/java/com/google/devtools/build/android/desugar/BUILD b/src/test/java/com/google/devtools/build/android/desugar/BUILD index daae6920d3..6487a8ee59 100644 --- a/src/test/java/com/google/devtools/build/android/desugar/BUILD +++ b/src/test/java/com/google/devtools/build/android/desugar/BUILD @@ -1620,7 +1620,7 @@ $(JAVABASE)/bin/javap -c -p -cp $< 'com.google.devtools.build.android.desugar.te tags = ["no_windows"], # Bazel 0.8.0 doesn't have this target under @bazel_tools, so we have to # use the one in the main repository - toolchains = ["@bazel_tools//tools/jdk:current_java_runtime"], + toolchains = ["@bazel_tools//tools/jdk:current_host_java_runtime"], tools = ["@bazel_tools//tools/jdk"], ) -- cgit v1.2.3