diff options
author | Miguel Alcon Pinto <malcon@google.com> | 2015-11-18 15:38:33 +0000 |
---|---|---|
committer | Damien Martin-Guillerez <dmarting@google.com> | 2015-11-19 09:58:45 +0000 |
commit | 8e7da8704dba78dba47fbaa0eafce3a91c2deb13 (patch) | |
tree | 2de98139c95f4b1d51741c30cf190f617df2d72a /src/main/java/com/google/devtools/build/lib/query2/output | |
parent | 54f1562b60b6839adddbfc4706c0190c8aa48995 (diff) |
Allow output formatters to work in stream mode.
--
MOS_MIGRATED_REVID=108142169
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib/query2/output')
4 files changed, 222 insertions, 152 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/query2/output/OutputFormatter.java b/src/main/java/com/google/devtools/build/lib/query2/output/OutputFormatter.java index c773ed4003..1f7cbe9449 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/output/OutputFormatter.java +++ b/src/main/java/com/google/devtools/build/lib/query2/output/OutputFormatter.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.collect.CompactHashSet; import com.google.devtools.build.lib.events.Location; import com.google.devtools.build.lib.graph.Digraph; import com.google.devtools.build.lib.graph.Node; @@ -27,6 +28,7 @@ import com.google.devtools.build.lib.packages.Attribute; import com.google.devtools.build.lib.packages.PackageSerializer; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.query2.engine.OutputFormatterCallback; import com.google.devtools.build.lib.query2.output.QueryOptions.OrderOutput; import com.google.devtools.build.lib.syntax.EvalUtils; import com.google.devtools.build.lib.syntax.Printer; @@ -153,17 +155,20 @@ public abstract class OutputFormatter implements Serializable { AspectResolver aspectProvider) throws IOException, InterruptedException; /** - * Unordered output formatter (wrt. dependency ordering). + * Unordered streamed output formatter (wrt. dependency ordering). * - * <p>Formatters that support unordered output may be used when only the set of query results is + * <p>Formatters that support streamed output may be used when only the set of query results is * requested but their ordering is irrelevant. * - * <p>The benefit of using a unordered formatter is that we can save the potentially expensive - * subgraph extraction step before presenting the query results. + * <p>The benefit of using a streamed formatter is that we can save the potentially expensive + * subgraph extraction step before presenting the query results and that depending on the query + * environment used, it can be more memory performant, as it does not aggregate all the data + * before writting in the output. */ - public interface UnorderedFormatter { - void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out, - AspectResolver aspectResolver) throws IOException, InterruptedException; + public interface StreamedFormatter { + + OutputFormatterCallback<Target> createStreamCallback(QueryOptions options, PrintStream out, + AspectResolver aspectResolver); } /** @@ -172,8 +177,8 @@ public abstract class OutputFormatter implements Serializable { public abstract String getName(); abstract static class AbstractUnorderedFormatter extends OutputFormatter - implements UnorderedFormatter { - private static Iterable<Target> getOrderedTargets( + implements StreamedFormatter { + protected Iterable<Target> getOrderedTargets( Digraph<Target> result, QueryOptions options) { Iterable<Node<Target>> orderedResult = options.orderOutput == OrderOutput.DEPS @@ -183,13 +188,11 @@ public abstract class OutputFormatter implements Serializable { } @Override - public void output( - QueryOptions options, - Digraph<Target> result, - PrintStream out, - AspectResolver aspectResolver) - throws IOException, InterruptedException { - outputUnordered(options, getOrderedTargets(result, options), out, aspectResolver); + public void output(QueryOptions options, Digraph<Target> result, PrintStream out, + AspectResolver aspectResolver) throws IOException, InterruptedException { + OutputFormatterCallback.processAllTargets( + createStreamCallback(options, out, aspectResolver), + getOrderedTargets(result, options)); } } @@ -201,7 +204,7 @@ public abstract class OutputFormatter implements Serializable { private final boolean showKind; - public LabelOutputFormatter(boolean showKind) { + private LabelOutputFormatter(boolean showKind) { this.showKind = showKind; } @@ -211,15 +214,22 @@ public abstract class OutputFormatter implements Serializable { } @Override - public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out, - AspectResolver aspectResolver) { - for (Target target : result) { - if (showKind) { - out.print(target.getTargetKind()); - out.print(' '); + public OutputFormatterCallback<Target> createStreamCallback(QueryOptions options, + final PrintStream out, AspectResolver aspectResolver) { + return new OutputFormatterCallback<Target>() { + + @Override + protected void processOutput(Iterable<Target> partialResult) + throws IOException, InterruptedException { + for (Target target : partialResult) { + if (showKind) { + out.print(target.getTargetKind()); + out.print(' '); + } + out.println(target.getLabel()); + } } - out.println(target.getLabel()); - } + }; } } @@ -245,15 +255,28 @@ public abstract class OutputFormatter implements Serializable { } @Override - public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out, + public OutputFormatterCallback<Target> createStreamCallback(QueryOptions options, + final PrintStream out, AspectResolver aspectResolver) { - Set<String> packageNames = Sets.newTreeSet(); - for (Target target : result) { - packageNames.add(target.getLabel().getPackageName()); - } - for (String packageName : packageNames) { - out.println(packageName); - } + return new OutputFormatterCallback<Target>() { + private final Set<String> packageNames = Sets.newTreeSet(); + + @Override + protected void processOutput(Iterable<Target> partialResult) + throws IOException, InterruptedException { + + for (Target target : partialResult) { + packageNames.add(target.getLabel().getPackageName()); + } + } + + @Override + public void close() throws IOException { + for (String packageName : packageNames) { + out.println(packageName); + } + } + }; } } @@ -270,12 +293,20 @@ public abstract class OutputFormatter implements Serializable { } @Override - public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out, + public OutputFormatterCallback<Target> createStreamCallback(QueryOptions options, + final PrintStream out, AspectResolver aspectResolver) { - for (Target target : result) { - Location location = target.getLocation(); - out.println(location.print() + ": " + target.getTargetKind() + " " + target.getLabel()); - } + return new OutputFormatterCallback<Target>() { + + @Override + protected void processOutput(Iterable<Target> partialResult) + throws IOException, InterruptedException { + for (Target target : partialResult) { + Location location = target.getLocation(); + out.println(location.print() + ": " + target.getTargetKind() + " " + target.getLabel()); + } + } + }; } } @@ -290,48 +321,57 @@ public abstract class OutputFormatter implements Serializable { return "build"; } - private void outputRule(Rule rule, PrintStream out) { - out.printf("# %s%n", rule.getLocation()); - out.printf("%s(%n", rule.getRuleClass()); - out.printf(" name = \"%s\",%n", rule.getName()); - - for (Attribute attr : rule.getAttributes()) { - Pair<Iterable<Object>, AttributeValueSource> values = getAttributeValues(rule, attr); - if (Iterables.size(values.first) != 1) { - continue; // TODO(bazel-team): handle configurable attributes. - } - if (values.second != AttributeValueSource.RULE) { - continue; // Don't print default values. - } - Object value = Iterables.getOnlyElement(values.first); - out.printf(" %s = ", attr.getPublicName()); - if (value instanceof Label) { - value = value.toString(); - } else if (value instanceof List<?> && EvalUtils.isImmutable(value)) { - // Display it as a list (and not as a tuple). Attributes can never be tuples. - value = new ArrayList<>((List<?>) value); - } - // It is *much* faster to write to a StringBuilder compared to the PrintStream object. - StringBuilder builder = new StringBuilder(); - Printer.write(builder, value); - out.print(builder); - out.println(","); - } - out.printf(")\n%n"); - } - @Override - public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out, + public OutputFormatterCallback<Target> createStreamCallback(QueryOptions options, + final PrintStream out, AspectResolver aspectResolver) { - Set<Label> printed = new HashSet<>(); - for (Target target : result) { - Rule rule = target.getAssociatedRule(); - if (rule == null || printed.contains(rule.getLabel())) { - continue; + return new OutputFormatterCallback<Target>() { + private final Set<Label> printed = CompactHashSet.create(); + + private void outputRule(Rule rule, PrintStream out) { + out.printf("# %s%n", rule.getLocation()); + out.printf("%s(%n", rule.getRuleClass()); + out.printf(" name = \"%s\",%n", rule.getName()); + + for (Attribute attr : rule.getAttributes()) { + Pair<Iterable<Object>, AttributeValueSource> values = getAttributeValues(rule, attr); + if (Iterables.size(values.first) != 1) { + continue; // TODO(bazel-team): handle configurable attributes. + } + if (values.second != AttributeValueSource.RULE) { + continue; // Don't print default values. + } + Object value = Iterables.getOnlyElement(values.first); + out.printf(" %s = ", attr.getPublicName()); + if (value instanceof Label) { + value = value.toString(); + } else if (value instanceof List<?> && EvalUtils.isImmutable(value)) { + // Display it as a list (and not as a tuple). Attributes can never be tuples. + value = new ArrayList<>((List<?>) value); + } + // It is *much* faster to write to a StringBuilder compared to the PrintStream object. + StringBuilder builder = new StringBuilder(); + Printer.write(builder, value); + out.print(builder); + out.println(","); + } + out.printf(")\n%n"); } - outputRule(rule, out); - printed.add(rule.getLabel()); - } + + @Override + protected void processOutput(Iterable<Target> partialResult) + throws IOException, InterruptedException { + + for (Target target : partialResult) { + Rule rule = target.getAssociatedRule(); + if (rule == null || printed.contains(rule.getLabel())) { + continue; + } + outputRule(rule, out); + printed.add(rule.getLabel()); + } + } + }; } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/output/ProtoOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/query2/output/ProtoOutputFormatter.java index 66348b3be3..8bca7dfc1d 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/output/ProtoOutputFormatter.java +++ b/src/main/java/com/google/devtools/build/lib/query2/output/ProtoOutputFormatter.java @@ -34,10 +34,12 @@ import com.google.devtools.build.lib.packages.ProtoUtils; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.query2.FakeSubincludeTarget; +import com.google.devtools.build.lib.query2.engine.OutputFormatterCallback; import com.google.devtools.build.lib.query2.output.AspectResolver.BuildFileDependencyMode; -import com.google.devtools.build.lib.query2.output.OutputFormatter.UnorderedFormatter; +import com.google.devtools.build.lib.query2.output.OutputFormatter.AbstractUnorderedFormatter; import com.google.devtools.build.lib.query2.output.QueryOptions.OrderOutput; import com.google.devtools.build.lib.query2.proto.proto2api.Build; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.QueryResult.Builder; import com.google.devtools.build.lib.syntax.Environment; import com.google.devtools.build.lib.util.BinaryPredicate; @@ -54,7 +56,7 @@ import java.util.Set; * By taking the bytes and calling {@code mergeFrom()} on a * {@code Build.QueryResult} object the full result can be reconstructed. */ -public class ProtoOutputFormatter extends OutputFormatter implements UnorderedFormatter { +public class ProtoOutputFormatter extends AbstractUnorderedFormatter { /** * A special attribute name for the rule implementation hash code. @@ -77,19 +79,36 @@ public class ProtoOutputFormatter extends OutputFormatter implements UnorderedFo } @Override - public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out, - AspectResolver aspectResolver) throws IOException, InterruptedException { + public OutputFormatterCallback<Target> createStreamCallback(QueryOptions options, + final PrintStream out, AspectResolver aspectResolver) { relativeLocations = options.relativeLocations; this.aspectResolver = aspectResolver; this.includeDefaultValues = options.protoIncludeDefaultValues; setDependencyFilter(options); - Build.QueryResult.Builder queryResult = Build.QueryResult.newBuilder(); - for (Target target : result) { - addTarget(queryResult, target); - } + return new OutputFormatterCallback<Target>() { + + private Builder queryResult; + + @Override + public void start() { + queryResult = Build.QueryResult.newBuilder(); + } + + @Override + protected void processOutput(Iterable<Target> partialResult) + throws IOException, InterruptedException { + + for (Target target : partialResult) { + queryResult.addTarget(toTargetProtoBuffer(target)); + } + } - queryResult.build().writeTo(out); + @Override + public void close() throws IOException { + queryResult.build().writeTo(out); + } + }; } private static Iterable<Target> getSortedLabels(Digraph<Target> result) { @@ -98,24 +117,8 @@ public class ProtoOutputFormatter extends OutputFormatter implements UnorderedFo } @Override - public void output(QueryOptions options, Digraph<Target> result, PrintStream out, - AspectResolver aspectResolver) throws IOException, InterruptedException { - outputUnordered( - options, - options.orderOutput == OrderOutput.FULL ? getSortedLabels(result) : result.getLabels(), - out, - aspectResolver); - } - - /** - * Add the target to the query result. - * @param queryResult The query result that contains all rule, input and - * output targets. - * @param target The query target being converted to a protocol buffer. - */ - private void addTarget(Build.QueryResult.Builder queryResult, Target target) - throws InterruptedException { - queryResult.addTarget(toTargetProtoBuffer(target)); + protected Iterable<Target> getOrderedTargets(Digraph<Target> result, QueryOptions options) { + return options.orderOutput == OrderOutput.FULL ? getSortedLabels(result) : result.getLabels(); } /** diff --git a/src/main/java/com/google/devtools/build/lib/query2/output/QueryOutputUtils.java b/src/main/java/com/google/devtools/build/lib/query2/output/QueryOutputUtils.java index 36f5464461..1533b7bf43 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/output/QueryOutputUtils.java +++ b/src/main/java/com/google/devtools/build/lib/query2/output/QueryOutputUtils.java @@ -14,32 +14,43 @@ package com.google.devtools.build.lib.query2.output; import com.google.devtools.build.lib.packages.Target; -import com.google.devtools.build.lib.query2.engine.BlazeQueryEvalResult; +import com.google.devtools.build.lib.query2.engine.DigraphQueryEvalResult; +import com.google.devtools.build.lib.query2.engine.OutputFormatterCallback; import com.google.devtools.build.lib.query2.engine.QueryEvalResult; -import com.google.devtools.build.lib.query2.output.OutputFormatter.UnorderedFormatter; +import com.google.devtools.build.lib.query2.output.OutputFormatter.StreamedFormatter; import com.google.devtools.build.lib.query2.output.QueryOptions.OrderOutput; import java.io.IOException; import java.io.PrintStream; +import java.util.Set; /** Static utility methods for outputting a query. */ public class QueryOutputUtils { // Utility class cannot be instantiated. private QueryOutputUtils() {} - public static boolean orderResults(QueryOptions queryOptions, OutputFormatter formatter) { - return queryOptions.orderOutput != OrderOutput.NO || !(formatter instanceof UnorderedFormatter); + public static boolean shouldStreamResults(QueryOptions queryOptions, OutputFormatter formatter) { + return queryOptions.orderOutput == OrderOutput.NO + && formatter instanceof StreamedFormatter; } - public static void output(QueryOptions queryOptions, QueryEvalResult<Target> result, - OutputFormatter formatter, PrintStream outputStream, AspectResolver aspectResolver) + public static void output(QueryOptions queryOptions, QueryEvalResult result, + Set<Target> targetsResult, OutputFormatter formatter, PrintStream outputStream, + AspectResolver aspectResolver) throws IOException, InterruptedException { - if (orderResults(queryOptions, formatter)) { - formatter.output(queryOptions, ((BlazeQueryEvalResult<Target>) result).getResultGraph(), + /* + * This is not really streaming, but we are using the streaming interface for writing into the + * output everything in one batch. This happens when the QueryEnvironment does not + * support streaming but we don't care about ordered results. + */ + boolean orderedResults = !shouldStreamResults(queryOptions, formatter); + if (orderedResults) { + formatter.output(queryOptions, + ((DigraphQueryEvalResult<Target>) result).getGraph().extractSubgraph(targetsResult), outputStream, aspectResolver); } else { - ((UnorderedFormatter) formatter).outputUnordered(queryOptions, result.getResultSet(), - outputStream, aspectResolver); + OutputFormatterCallback.processAllTargets(((StreamedFormatter) formatter) + .createStreamCallback(queryOptions, outputStream, aspectResolver), targetsResult); } } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/output/XmlOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/query2/output/XmlOutputFormatter.java index b8b4466bad..2af71b7414 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/output/XmlOutputFormatter.java +++ b/src/main/java/com/google/devtools/build/lib/query2/output/XmlOutputFormatter.java @@ -27,6 +27,7 @@ import com.google.devtools.build.lib.packages.PackageGroup; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.query2.FakeSubincludeTarget; +import com.google.devtools.build.lib.query2.engine.OutputFormatterCallback; import com.google.devtools.build.lib.query2.output.AspectResolver.BuildFileDependencyMode; import com.google.devtools.build.lib.query2.output.OutputFormatter.AbstractUnorderedFormatter; import com.google.devtools.build.lib.util.BinaryPredicate; @@ -36,6 +37,7 @@ import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; +import java.io.IOException; import java.io.PrintStream; import java.util.Collection; import java.util.HashSet; @@ -57,11 +59,9 @@ import javax.xml.transform.stream.StreamResult; */ class XmlOutputFormatter extends AbstractUnorderedFormatter { - private boolean xmlLineNumbers; - private boolean showDefaultValues; - private boolean relativeLocations; - private transient AspectResolver aspectResolver; - private transient BinaryPredicate<Rule, Attribute> dependencyFilter; + private QueryOptions options; + private AspectResolver aspectResolver; + private BinaryPredicate<Rule, Attribute> dependencyFilter; @Override public String getName() { @@ -69,36 +69,52 @@ class XmlOutputFormatter extends AbstractUnorderedFormatter { } @Override - public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out, - AspectResolver aspectResolver) throws InterruptedException { - this.xmlLineNumbers = options.xmlLineNumbers; - this.showDefaultValues = options.xmlShowDefaultValues; - this.relativeLocations = options.relativeLocations; - this.dependencyFilter = OutputFormatter.getDependencyFilter(options); + public OutputFormatterCallback<Target> createStreamCallback(QueryOptions options, + final PrintStream out, AspectResolver aspectResolver) { + this.options = options; this.aspectResolver = aspectResolver; - Document doc; - try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - doc = factory.newDocumentBuilder().newDocument(); - } catch (ParserConfigurationException e) { - // This shouldn't be possible: all the configuration is hard-coded. - throw new IllegalStateException("XML output failed", e); - } - doc.setXmlVersion("1.1"); - Element queryElem = doc.createElement("query"); - queryElem.setAttribute("version", "2"); - doc.appendChild(queryElem); - for (Target target : result) { - queryElem.appendChild(createTargetElement(doc, target)); - } - try { - Transformer transformer = TransformerFactory.newInstance().newTransformer(); - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - transformer.transform(new DOMSource(doc), new StreamResult(out)); - } catch (TransformerFactoryConfigurationError | TransformerException e) { - // This shouldn't be possible: all the configuration is hard-coded. - throw new IllegalStateException("XML output failed", e); - } + this.dependencyFilter = OutputFormatter.getDependencyFilter(options); + return new OutputFormatterCallback<Target>() { + + private Document doc; + private Element queryElem; + + + @Override + public void start() { + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + doc = factory.newDocumentBuilder().newDocument(); + } catch (ParserConfigurationException e) { + // This shouldn't be possible: all the configuration is hard-coded. + throw new IllegalStateException("XML output failed", e); + } + doc.setXmlVersion("1.1"); + queryElem = doc.createElement("query"); + queryElem.setAttribute("version", "2"); + doc.appendChild(queryElem); + } + + @Override + protected void processOutput(Iterable<Target> partialResult) + throws IOException, InterruptedException { + for (Target target : partialResult) { + queryElem.appendChild(createTargetElement(doc, target)); + } + } + + @Override + public void close() throws IOException { + try { + Transformer transformer = TransformerFactory.newInstance().newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.transform(new DOMSource(doc), new StreamResult(out)); + } catch (TransformerFactoryConfigurationError | TransformerException e) { + // This shouldn't be possible: all the configuration is hard-coded. + throw new IllegalStateException("XML output failed", e); + } + } + }; } /** @@ -120,9 +136,9 @@ class XmlOutputFormatter extends AbstractUnorderedFormatter { Rule rule = (Rule) target; elem = doc.createElement("rule"); elem.setAttribute("class", rule.getRuleClass()); - for (Attribute attr: rule.getAttributes()) { + for (Attribute attr : rule.getAttributes()) { Pair<Iterable<Object>, AttributeValueSource> values = getAttributeValues(rule, attr); - if (values.second == AttributeValueSource.RULE || showDefaultValues) { + if (values.second == AttributeValueSource.RULE || options.xmlShowDefaultValues) { Element attrElem = createValueElement(doc, attr.getType(), values.first); attrElem.setAttribute("name", attr.getName()); elem.appendChild(attrElem); @@ -205,8 +221,8 @@ class XmlOutputFormatter extends AbstractUnorderedFormatter { } elem.setAttribute("name", target.getLabel().toString()); - String location = getLocation(target, relativeLocations); - if (!xmlLineNumbers) { + String location = getLocation(target, options.relativeLocations); + if (!options.xmlLineNumbers) { int firstColon = location.indexOf(':'); if (firstColon != -1) { location = location.substring(0, firstColon); |