Use Apache Velocity for templating for generating Build Encyclopedia. Refactor code used for extracting documentation from rule classes into separate class.

Todo after this CL:
* Use RuleDocumentation and RuleDocumentationAttribute directly in .vm templates to further simplify BuildEncyclopediaProcessor code
* Use VTL #include for including common attribute documentation
* Split BE into multiple pages
* Use templating for Skylark Library

--
MOS_MIGRATED_REVID=96865477

9 files changed, 341 insertions, 183 deletions

diff --git a/src/main/java/com/google/devtools/build/docgen/BlazeRuleHelpPrinter.java b/src/main/java/com/google/devtools/build/docgen/BlazeRuleHelpPrinter.java
index cec65024ef..ed0e415268 100644
--- a/src/main/java/com/google/devtools/build/docgen/BlazeRuleHelpPrinter.java
+++ b/src/main/java/com/google/devtools/build/docgen/BlazeRuleHelpPrinter.java
@@ -35,10 +35,10 @@ public class BlazeRuleHelpPrinter {
   public static String getRuleDoc(String ruleName, ConfiguredRuleClassProvider provider) {
     if (ruleDocMap == null) {
       try {
-        BuildEncyclopediaProcessor processor = new BuildEncyclopediaProcessor(provider);
-        Map<String, RuleDocumentation> ruleDocs = processor.collectAndProcessRuleDocs(
+        BuildDocCollector collector = new BuildDocCollector(provider, false);
+        Map<String, RuleDocumentation> ruleDocs = collector.collect(
             new String[] {"java/com/google/devtools/build/lib/view",
-                "java/com/google/devtools/build/lib/rules"}, false);
+                "java/com/google/devtools/build/lib/rules"});
         ruleDocMap = new HashMap<>();
         for (RuleDocumentation ruleDoc : ruleDocs.values()) {
           ruleDocMap.put(ruleDoc.getRuleName(), ruleDoc);
diff --git a/src/main/java/com/google/devtools/build/docgen/BuildDocCollector.java b/src/main/java/com/google/devtools/build/docgen/BuildDocCollector.java
new file mode 100644
index 0000000000..0bf4f20030
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/BuildDocCollector.java
@@ -0,0 +1,184 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.docgen;
+
+import com.google.common.collect.LinkedListMultimap;
+import com.google.common.collect.ListMultimap;
+import com.google.devtools.build.docgen.DocgenConsts.RuleType;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.RuleClass;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+/**
+ * Class that parses the documentation fragments of rule-classes and
+ * generates the html format documentation.
+ */
+class BuildDocCollector {
+  private ConfiguredRuleClassProvider ruleClassProvider;
+  private boolean printMessages;
+
+  public BuildDocCollector(ConfiguredRuleClassProvider ruleClassProvider,
+      boolean printMessages) {
+    this.ruleClassProvider = ruleClassProvider;
+    this.printMessages = printMessages;
+  }
+
+  /**
+   * Collects all the rule and attribute documentation present in inputDirs, integrates the
+   * attribute documentation in the rule documentation and returns the rule documentation.
+   */
+  public Map<String, RuleDocumentation> collect(String[] inputDirs)
+      throws BuildEncyclopediaDocException, IOException {
+    // RuleDocumentations are generated in order (based on rule type then alphabetically).
+    // The ordering is also used to determine in which rule doc the common attribute docs are
+    // generated (they are generated at the first appearance).
+    Map<String, RuleDocumentation> ruleDocEntries = new TreeMap<>();
+    // RuleDocumentationAttribute objects equal based on attributeName so they have to be
+    // collected in a List instead of a Set.
+    ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries =
+        LinkedListMultimap.create();
+
+    // Map of rule class name to file that defined it.
+    Map<String, File> ruleClassFiles = new HashMap<>();
+
+    // Set of files already processed. The same file may be encountered multiple times because
+    // directories are processed recursively, and an input directory may be a subdirectory of
+    // another one.
+    Set<File> processedFiles = new HashSet<>();
+
+    for (String inputDir : inputDirs) {
+      if (printMessages) {
+        System.out.println(" Processing input directory: " + inputDir);
+      }
+      int ruleNum = ruleDocEntries.size();
+      collectDocs(processedFiles, ruleClassFiles, ruleDocEntries, attributeDocEntries,
+          new File(inputDir));
+      if (printMessages) {
+        System.out.println(" " + (ruleDocEntries.size() - ruleNum)
+            + " rule documentations found.");
+      }
+    }
+
+    processAttributeDocs(ruleDocEntries.values(), attributeDocEntries);
+    return ruleDocEntries;
+  }
+
+  /**
+   * Go through all attributes of all documented rules and search the best attribute documentation
+   * if exists. The best documentation is the closest documentation in the ancestor graph. E.g. if
+   * java_library.deps documented in $rule and $java_rule then the one in $java_rule is going to
+   * apply since it's a closer ancestor of java_library.
+   */
+  private void processAttributeDocs(Iterable<RuleDocumentation> ruleDocEntries,
+      ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries)
+          throws BuildEncyclopediaDocException {
+    for (RuleDocumentation ruleDoc : ruleDocEntries) {
+      RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleDoc.getRuleName());
+      if (ruleClass != null) {
+        if (ruleClass.isDocumented()) {
+          Class<? extends RuleDefinition> ruleDefinition =
+              ruleClassProvider.getRuleClassDefinition(ruleDoc.getRuleName());
+          for (Attribute attribute : ruleClass.getAttributes()) {
+            String attrName = attribute.getName();
+            List<RuleDocumentationAttribute> attributeDocList =
+                attributeDocEntries.get(attrName);
+            if (attributeDocList != null) {
+              // There are attribute docs for this attribute.
+              // Search the closest one in the ancestor graph.
+              // Note that there can be only one 'closest' attribute since we forbid multiple
+              // inheritance of the same attribute in RuleClass.
+              int minLevel = Integer.MAX_VALUE;
+              RuleDocumentationAttribute bestAttributeDoc = null;
+              for (RuleDocumentationAttribute attributeDoc : attributeDocList) {
+                int level = attributeDoc.getDefinitionClassAncestryLevel(ruleDefinition);
+                if (level >= 0 && level < minLevel) {
+                  bestAttributeDoc = attributeDoc;
+                  minLevel = level;
+                }
+              }
+              if (bestAttributeDoc != null) {
+                ruleDoc.addAttribute(bestAttributeDoc);
+              // If there is no matching attribute doc try to add the common.
+              } else if (ruleDoc.getRuleType().equals(RuleType.BINARY)
+                  && PredefinedAttributes.BINARY_ATTRIBUTES.containsKey(attrName)) {
+                ruleDoc.addAttribute(PredefinedAttributes.BINARY_ATTRIBUTES.get(attrName));
+              } else if (ruleDoc.getRuleType().equals(RuleType.TEST)
+                  && PredefinedAttributes.TEST_ATTRIBUTES.containsKey(attrName)) {
+                ruleDoc.addAttribute(PredefinedAttributes.TEST_ATTRIBUTES.get(attrName));
+              } else if (PredefinedAttributes.COMMON_ATTRIBUTES.containsKey(attrName)) {
+                ruleDoc.addAttribute(PredefinedAttributes.COMMON_ATTRIBUTES.get(attrName));
+              }
+            }
+          }
+        }
+      } else {
+        throw ruleDoc.createException("Can't find RuleClass for " + ruleDoc.getRuleName());
+      }
+    }
+  }
+
+  /**
+   * Goes through all the html files and subdirs under inputPath and collects the rule
+   * and attribute documentations using the ruleDocEntries and attributeDocEntries variable.
+   */
+  public void collectDocs(
+      Set<File> processedFiles,
+      Map<String, File> ruleClassFiles,
+      Map<String, RuleDocumentation> ruleDocEntries,
+      ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries,
+      File inputPath) throws BuildEncyclopediaDocException, IOException {
+    if (processedFiles.contains(inputPath)) {
+      return;
+    }
+
+    if (inputPath.isFile()) {
+      if (DocgenConsts.JAVA_SOURCE_FILE_SUFFIX.apply(inputPath.getName())) {
+        SourceFileReader sfr = new SourceFileReader(
+            ruleClassProvider, inputPath.getAbsolutePath());
+        sfr.readDocsFromComments();
+        for (RuleDocumentation d : sfr.getRuleDocEntries()) {
+          String ruleName = d.getRuleName();
+          if (ruleDocEntries.containsKey(ruleName)
+              && !ruleClassFiles.get(ruleName).equals(inputPath)) {
+            System.err.printf("WARNING: '%s' from '%s' overrides value already in map from '%s'\n",
+                d.getRuleName(), inputPath, ruleClassFiles.get(ruleName));
+          }
+          ruleClassFiles.put(ruleName, inputPath);
+          ruleDocEntries.put(ruleName, d);
+        }
+        if (attributeDocEntries != null) {
+          // Collect all attribute documentations from this file.
+          attributeDocEntries.putAll(sfr.getAttributeDocEntries());
+        }
+      }
+    } else if (inputPath.isDirectory()) {
+      for (File childPath : inputPath.listFiles()) {
+        collectDocs(processedFiles, ruleClassFiles, ruleDocEntries, attributeDocEntries, childPath);
+      }
+    }
+
+    processedFiles.add(inputPath);
+  }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java
index 4100b1daa0..4990d97c08 100644
--- a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java
+++ b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java
@@ -11,13 +11,12 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 package com.google.devtools.build.docgen;
 
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Predicate;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableMap.Builder;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.LinkedListMultimap;
@@ -26,16 +25,11 @@ import com.google.common.collect.Ordering;
 import com.google.common.collect.Sets;
 import com.google.devtools.build.docgen.DocgenConsts.RuleType;
 import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
-import com.google.devtools.build.lib.analysis.RuleDefinition;
-import com.google.devtools.build.lib.packages.Attribute;
 import com.google.devtools.build.lib.packages.RuleClass;
 
-import java.io.BufferedWriter;
 import java.io.File;
-import java.io.FileWriter;
 import java.io.IOException;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -43,9 +37,8 @@ import java.util.TreeMap;
 import java.util.TreeSet;
 
 /**
- * A class to assemble documentation for the Build Encyclopedia. The
- * program parses the documentation fragments of rule-classes and
- * generates the html format documentation.
+ * A class to assemble documentation for the Build Encyclopedia. This class uses
+ * {@link BuildDocCollector} to extract documentation fragments from rule classes.
  */
 public class BuildEncyclopediaProcessor {
   private static final Predicate<String> RULE_WORTH_DOCUMENTING = new Predicate<String>() {
@@ -73,118 +66,23 @@ public class BuildEncyclopediaProcessor {
   public void generateDocumentation(String[] inputDirs, String outputRootDir)
       throws BuildEncyclopediaDocException, IOException {
     File buildEncyclopediaPath = setupDirectories(outputRootDir);
-    try (BufferedWriter bw = new BufferedWriter(new FileWriter(buildEncyclopediaPath))) {
-      bw.write(DocgenConsts.HEADER_COMMENT);
-      bw.write("\n");  // for the benefit of the block-beginning comment at the top of the template
 
-      Map<String, RuleDocumentation> ruleDocEntries = collectAndProcessRuleDocs(inputDirs, false);
-      warnAboutUndocumentedRules(
-          Sets.difference(ruleClassProvider.getRuleClassMap().keySet(), ruleDocEntries.keySet()));
-      writeRuleClassDocs(ruleDocEntries.values(), bw);
+    Page page = TemplateEngine.newPage(
+        "com/google/devtools/build/docgen/templates/build-encyclopedia.vm");
 
-      bw.write("\n");  // for the benefit of the block-beginning comment at the top of the template
-      bw.write(SourceFileReader.readTemplateContents(DocgenConsts.FOOTER_TEMPLATE));
-    }
+    BuildDocCollector collector = new BuildDocCollector(ruleClassProvider, false);
+    Map<String, RuleDocumentation> ruleDocEntries = collector.collect(inputDirs);
+    warnAboutUndocumentedRules(
+        Sets.difference(ruleClassProvider.getRuleClassMap().keySet(), ruleDocEntries.keySet()));
+    writeRuleClassDocs(ruleDocEntries.values(), page);
+    page.write(buildEncyclopediaPath);
   }
 
-  /**
-   * Collects all the rule and attribute documentation present in inputDirs, integrates the
-   * attribute documentation in the rule documentation and returns the rule documentation.
-   */
-  public Map<String, RuleDocumentation> collectAndProcessRuleDocs(String[] inputDirs,
-      boolean printMessages) throws BuildEncyclopediaDocException, IOException {
-    // RuleDocumentations are generated in order (based on rule type then alphabetically).
-    // The ordering is also used to determine in which rule doc the common attribute docs are
-    // generated (they are generated at the first appearance).
-    Map<String, RuleDocumentation> ruleDocEntries = new TreeMap<>();
-    // RuleDocumentationAttribute objects equal based on attributeName so they have to be
-    // collected in a List instead of a Set.
-    ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries =
-        LinkedListMultimap.create();
-
-    // Map of rule class name to file that defined it.
-    Map<String, File> ruleClassFiles = new HashMap<>();
-
-    // Set of files already processed. The same file may be encountered multiple times because
-    // directories are processed recursively, and an input directory may be a subdirectory of
-    // another one.
-    Set<File> processedFiles = new HashSet<>();
-
-    for (String inputDir : inputDirs) {
-      if (printMessages) {
-        System.out.println(" Processing input directory: " + inputDir);
-      }
-      int ruleNum = ruleDocEntries.size();
-      collectDocs(processedFiles, ruleClassFiles, ruleDocEntries, attributeDocEntries,
-          new File(inputDir));
-      if (printMessages) {
-        System.out.println(
-          " " + (ruleDocEntries.size() - ruleNum) + " rule documentations found.");
-      }
-    }
-
-    processAttributeDocs(ruleDocEntries.values(), attributeDocEntries);
-    return ruleDocEntries;
-  }
-
-  /**
-   * Go through all attributes of all documented rules and search the best attribute documentation
-   * if exists. The best documentation is the closest documentation in the ancestor graph. E.g. if
-   * java_library.deps documented in $rule and $java_rule then the one in $java_rule is going to
-   * apply since it's a closer ancestor of java_library.
-   */
-  private void processAttributeDocs(Iterable<RuleDocumentation> ruleDocEntries,
-      ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries)
-          throws BuildEncyclopediaDocException {
-    for (RuleDocumentation ruleDoc : ruleDocEntries) {
-      RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleDoc.getRuleName());
-      if (ruleClass != null) {
-        if (ruleClass.isDocumented()) {
-          Class<? extends RuleDefinition> ruleDefinition =
-              ruleClassProvider.getRuleClassDefinition(ruleDoc.getRuleName());
-          for (Attribute attribute : ruleClass.getAttributes()) {
-            String attrName = attribute.getName();
-            List<RuleDocumentationAttribute> attributeDocList =
-                attributeDocEntries.get(attrName);
-            if (attributeDocList != null) {
-              // There are attribute docs for this attribute.
-              // Search the closest one in the ancestor graph.
-              // Note that there can be only one 'closest' attribute since we forbid multiple
-              // inheritance of the same attribute in RuleClass.
-              int minLevel = Integer.MAX_VALUE;
-              RuleDocumentationAttribute bestAttributeDoc = null;
-              for (RuleDocumentationAttribute attributeDoc : attributeDocList) {
-                int level = attributeDoc.getDefinitionClassAncestryLevel(ruleDefinition);
-                if (level >= 0 && level < minLevel) {
-                  bestAttributeDoc = attributeDoc;
-                  minLevel = level;
-                }
-              }
-              if (bestAttributeDoc != null) {
-                ruleDoc.addAttribute(bestAttributeDoc);
-              // If there is no matching attribute doc try to add the common.
-              } else if (ruleDoc.getRuleType().equals(RuleType.BINARY)
-                  && PredefinedAttributes.BINARY_ATTRIBUTES.containsKey(attrName)) {
-                ruleDoc.addAttribute(PredefinedAttributes.BINARY_ATTRIBUTES.get(attrName));
-              } else if (ruleDoc.getRuleType().equals(RuleType.TEST)
-                  && PredefinedAttributes.TEST_ATTRIBUTES.containsKey(attrName)) {
-                ruleDoc.addAttribute(PredefinedAttributes.TEST_ATTRIBUTES.get(attrName));
-              } else if (PredefinedAttributes.COMMON_ATTRIBUTES.containsKey(attrName)) {
-                ruleDoc.addAttribute(PredefinedAttributes.COMMON_ATTRIBUTES.get(attrName));
-              }
-            }
-          }
-        }
-      } else {
-        throw ruleDoc.createException("Can't find RuleClass for " + ruleDoc.getRuleName());
-      }
-    }
-  }
 
   /**
    * Categorizes, checks and prints all the rule-class documentations.
    */
-  private void writeRuleClassDocs(Iterable<RuleDocumentation> docEntries, BufferedWriter bw)
+  private void writeRuleClassDocs(Iterable<RuleDocumentation> docEntries, Page page)
       throws BuildEncyclopediaDocException, IOException {
     Set<RuleDocumentation> binaryDocs = new TreeSet<>();
     Set<RuleDocumentation> libraryDocs = new TreeSet<>();
@@ -217,20 +115,15 @@ public class BuildEncyclopediaProcessor {
       }
     }
 
-    bw.write("\n");  // for the benefit of the block-beginning comment at the top of the template
-    bw.write(SourceFileReader.readTemplateContents(DocgenConsts.HEADER_TEMPLATE,
-        generateBEHeaderMapping(docEntries)));
+    renderBeHeader(docEntries, page);
 
-    Map<String, String> sectionMapping = ImmutableMap.of(
-        DocgenConsts.VAR_SECTION_BINARY,   getRuleDocs(binaryDocs),
-        DocgenConsts.VAR_SECTION_LIBRARY,  getRuleDocs(libraryDocs),
-        DocgenConsts.VAR_SECTION_TEST,     getRuleDocs(testDocs),
-        DocgenConsts.VAR_SECTION_OTHER,    getRuleDocs(otherDocs));
-    bw.write("\n");  // for the benefit of the block-beginning comment at the top of the template
-    bw.write(SourceFileReader.readTemplateContents(DocgenConsts.BODY_TEMPLATE, sectionMapping));
+    page.add(DocgenConsts.VAR_SECTION_BINARY, getRuleDocs(binaryDocs));
+    page.add(DocgenConsts.VAR_SECTION_LIBRARY, getRuleDocs(libraryDocs));
+    page.add(DocgenConsts.VAR_SECTION_TEST, getRuleDocs(testDocs));
+    page.add(DocgenConsts.VAR_SECTION_OTHER, getRuleDocs(otherDocs));
   }
 
-  private Map<String, String> generateBEHeaderMapping(Iterable<RuleDocumentation> docEntries)
+  private void renderBeHeader(Iterable<RuleDocumentation> docEntries, Page page)
       throws BuildEncyclopediaDocException {
     // Separate rule families into language-specific and generic ones.
     Set<String> languageSpecificRuleFamilies = new TreeSet<>();
@@ -266,18 +159,18 @@ public class BuildEncyclopediaProcessor {
       }
       otherRulesTable = sb.toString();
     }
-
-    return new ImmutableMap.Builder<String, String>()
-        .put(DocgenConsts.VAR_LANG_SPECIFIC_HEADER_TABLE, languageSpecificTable)
-        .put(DocgenConsts.VAR_OTHER_RULES_HEADER_TABLE, otherRulesTable)
-        .put(DocgenConsts.VAR_COMMON_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs(
-            PredefinedAttributes.COMMON_ATTRIBUTES, DocgenConsts.COMMON_ATTRIBUTES))
-        .put(DocgenConsts.VAR_TEST_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs(
-            PredefinedAttributes.TEST_ATTRIBUTES, DocgenConsts.TEST_ATTRIBUTES))
-        .put(DocgenConsts.VAR_BINARY_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs(
-            PredefinedAttributes.BINARY_ATTRIBUTES, DocgenConsts.BINARY_ATTRIBUTES))
-        .put(DocgenConsts.VAR_LEFT_PANEL, generateLeftNavigationPanel(docEntries))
-        .build();
+    page.add(DocgenConsts.VAR_LANG_SPECIFIC_HEADER_TABLE, languageSpecificTable);
+    page.add(DocgenConsts.VAR_OTHER_RULES_HEADER_TABLE, otherRulesTable);
+    page.add(DocgenConsts.VAR_COMMON_ATTRIBUTE_DEFINITION,
+        generateCommonAttributeDocs(
+            PredefinedAttributes.COMMON_ATTRIBUTES, DocgenConsts.COMMON_ATTRIBUTES));
+    page.add(DocgenConsts.VAR_TEST_ATTRIBUTE_DEFINITION,
+        generateCommonAttributeDocs(
+            PredefinedAttributes.TEST_ATTRIBUTES, DocgenConsts.TEST_ATTRIBUTES));
+    page.add(DocgenConsts.VAR_BINARY_ATTRIBUTE_DEFINITION,
+        generateCommonAttributeDocs(
+            PredefinedAttributes.BINARY_ATTRIBUTES, DocgenConsts.BINARY_ATTRIBUTES));
+    page.add(DocgenConsts.VAR_LEFT_PANEL, generateLeftNavigationPanel(docEntries));
   }
 
   /**
@@ -398,49 +291,6 @@ public class BuildEncyclopediaProcessor {
     return sb.toString();
   }
 
-  /**
-   * Goes through all the html files and subdirs under inputPath and collects the rule
-   * and attribute documentations using the ruleDocEntries and attributeDocEntries variable.
-   */
-  public void collectDocs(
-      Set<File> processedFiles,
-      Map<String, File> ruleClassFiles,
-      Map<String, RuleDocumentation> ruleDocEntries,
-      ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries,
-      File inputPath) throws BuildEncyclopediaDocException, IOException {
-    if (processedFiles.contains(inputPath)) {
-      return;
-    }
-
-    if (inputPath.isFile()) {
-      if (DocgenConsts.JAVA_SOURCE_FILE_SUFFIX.apply(inputPath.getName())) {
-        SourceFileReader sfr = new SourceFileReader(
-            ruleClassProvider, inputPath.getAbsolutePath());
-        sfr.readDocsFromComments();
-        for (RuleDocumentation d : sfr.getRuleDocEntries()) {
-          String ruleName = d.getRuleName();
-          if (ruleDocEntries.containsKey(ruleName)
-              && !ruleClassFiles.get(ruleName).equals(inputPath)) {
-            System.err.printf("WARNING: '%s' from '%s' overrides value already in map from '%s'\n",
-                d.getRuleName(), inputPath, ruleClassFiles.get(ruleName));
-          }
-          ruleClassFiles.put(ruleName, inputPath);
-          ruleDocEntries.put(ruleName, d);
-        }
-        if (attributeDocEntries != null) {
-          // Collect all attribute documentations from this file.
-          attributeDocEntries.putAll(sfr.getAttributeDocEntries());
-        }
-      }
-    } else if (inputPath.isDirectory()) {
-      for (File childPath : inputPath.listFiles()) {
-        collectDocs(processedFiles, ruleClassFiles, ruleDocEntries, attributeDocEntries, childPath);
-      }
-    }
-
-    processedFiles.add(inputPath);
-  }
-
   private File setupDirectories(String outputRootDir) {
     if (outputRootDir != null) {
       File outputRootPath = new File(outputRootDir);
diff --git a/src/main/java/com/google/devtools/build/docgen/Page.java b/src/main/java/com/google/devtools/build/docgen/Page.java
new file mode 100644
index 0000000000..6cecb102be
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/Page.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.docgen;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.VelocityEngine;
+import org.apache.velocity.exception.MethodInvocationException;
+import org.apache.velocity.exception.ParseErrorException;
+import org.apache.velocity.exception.ResourceNotFoundException;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+
+/**
+ * Class that represents a page to be generated using the {@link TemplateEngine}.
+ */
+class Page {
+  private final VelocityEngine engine;
+  private final VelocityContext context;
+  private final String template;
+
+  /**
+   * Creates a new Page instance using the reference to the VelocityEngine and the .vm
+   * template file path.
+   */
+  public Page(VelocityEngine engine, String template) {
+    this.engine = engine;
+    this.template = template;
+    this.context = new VelocityContext();
+  }
+
+  /**
+   * Sets a Velocity variable in the template with the given value.
+   */
+  public void add(String var, Object value) {
+    context.put(var, value);
+  }
+
+  /**
+   * Renders the template and writes the output to the given file.
+   */
+  public void write(File outputFile) throws IOException {
+    OutputStream out = new FileOutputStream(outputFile);
+    try (Writer writer = new OutputStreamWriter(out, UTF_8)) {
+      engine.mergeTemplate(template, "UTF-8", context, writer);
+    } catch (ResourceNotFoundException|ParseErrorException|MethodInvocationException e) {
+      throw new IOException(e);
+    }
+  }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/TemplateEngine.java b/src/main/java/com/google/devtools/build/docgen/TemplateEngine.java
new file mode 100644
index 0000000000..f7f2dd9fed
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/TemplateEngine.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.docgen;
+
+import org.apache.velocity.app.VelocityEngine;
+import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader;
+import org.apache.velocity.runtime.resource.loader.JarResourceLoader;
+
+/**
+ * Utility class used for creating pages to be generated using VelcityEngine.
+ */
+final class TemplateEngine {
+  private TemplateEngine() {}
+
+  /**
+   * Returns a new {@link Page} using the given .vm template file path. The template file
+   * path must be the resource path for the .vm file in the JAR since the VelocityEngine
+   * is configured to load .vm files from JAR resources.
+   */
+  public static Page newPage(String template) {
+    VelocityEngine engine = new VelocityEngine();
+    engine.setProperty("resource.loader", "classpath, jar");
+    engine.setProperty("classpath.resource.loader.class", ClasspathResourceLoader.class.getName());
+    engine.setProperty("jar.resource.loader.class", JarResourceLoader.class.getName());
+    engine.setProperty("input.encoding", "UTF-8");
+    engine.setProperty("output.encoding", "UTF-8");
+    engine.setProperty("directive.set.null.allowed", true);
+    engine.setProperty("parser.pool.size", 3);
+    return new Page(engine, template);
+  }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/templates/be-footer.html b/src/main/java/com/google/devtools/build/docgen/templates/be-footer.vm
index 96b52aedb2..96b52aedb2 100644
--- a/src/main/java/com/google/devtools/build/docgen/templates/be-footer.html
+++ b/src/main/java/com/google/devtools/build/docgen/templates/be-footer.vm
diff --git a/src/main/java/com/google/devtools/build/docgen/templates/be-header.html b/src/main/java/com/google/devtools/build/docgen/templates/be-header.vm
index 8e841a626d..8e841a626d 100644
--- a/src/main/java/com/google/devtools/build/docgen/templates/be-header.html
+++ b/src/main/java/com/google/devtools/build/docgen/templates/be-header.vm
diff --git a/src/main/java/com/google/devtools/build/docgen/templates/be-body.html b/src/main/java/com/google/devtools/build/docgen/templates/build-encyclopedia.vm
index 0e5b01b270..aa7f4fa82f 100644
--- a/src/main/java/com/google/devtools/build/docgen/templates/be-body.html
+++ b/src/main/java/com/google/devtools/build/docgen/templates/build-encyclopedia.vm
@@ -1,3 +1,7 @@
+#parse("com/google/devtools/build/docgen/templates/header-comment.vm")
+
+#parse("com/google/devtools/build/docgen/templates/be-header.vm")
+
 <!-- ============================================
                       binary
      ============================================
@@ -238,3 +242,5 @@ in your genrule's cmd attribute.
 
 ${SECTION_OTHER}
 
+
+#parse("com/google/devtools/build/docgen/templates/be-footer.vm")
diff --git a/src/main/java/com/google/devtools/build/docgen/templates/header-comment.vm b/src/main/java/com/google/devtools/build/docgen/templates/header-comment.vm
new file mode 100644
index 0000000000..60504e8702
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/templates/header-comment.vm
@@ -0,0 +1,7 @@
+<!DOCTYPE html>
+<!--
+ This document is synchronized with Bazel releases.
+ To edit, submit changes to the Bazel source code.
+-->
+
+<!-- Generated by //src/main/java/com/google/devtools/build/docgen:build-encyclopedia.html -->