From 515b3ac67761c6f4e5f8a73068ebe34ec6cb44de Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Mon, 5 Jun 2017 14:21:25 -0700
Subject: Add Clutz to TensorBoard build

This is so we can get JavaScript protobufs. This CL also improves the
web_aspect and makes some peculiar Closure Compiler errors go away
relating to externs.

PiperOrigin-RevId: 158061198
---
 WORKSPACE                                          |   8 +-
 tensorflow/tensorboard/BUILD                       |  11 +
 tensorflow/tensorboard/clutz.d.ts                  |  19 ++
 tensorflow/tensorboard/defs.bzl                    |   7 +
 .../tensorboard/vulcanize/Vulcanize.java           |  73 ++++--
 tensorflow/tensorboard/vulcanize.bzl               |  24 +-
 tensorflow/tensorboard/web.bzl                     | 273 +++++++++++----------
 tensorflow/workspace.bzl                           |  25 +-
 third_party/clutz.BUILD                            |  44 ++++
 third_party/clutz.bzl                              |  77 ++++++
 10 files changed, 390 insertions(+), 171 deletions(-)
 create mode 100644 tensorflow/tensorboard/clutz.d.ts
 create mode 100644 third_party/clutz.BUILD
 create mode 100644 third_party/clutz.bzl

diff --git a/WORKSPACE b/WORKSPACE
index c9d7b458a9..74ce13f4e8 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -2,11 +2,11 @@ workspace(name = "org_tensorflow")
 
 http_archive(
     name = "io_bazel_rules_closure",
-    sha256 = "edc91f556b762fc5212d1050d00b12e40dd0b0b1c1d5d96886b59e9a30a6cae4",
-    strip_prefix = "rules_closure-3f07fb6a58870afbb36051bd5d54da4479561cc6",
+    sha256 = "bc41b80486413aaa551860fc37471dbc0666e1dbb5236fb6177cb83b0c105846",
+    strip_prefix = "rules_closure-dec425a4ff3faf09a56c85d082e4eed05d8ce38f",
     urls = [
-        "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/3f07fb6a58870afbb36051bd5d54da4479561cc6.tar.gz",  # 2017-05-31
-        "https://github.com/bazelbuild/rules_closure/archive/3f07fb6a58870afbb36051bd5d54da4479561cc6.tar.gz",
+        "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz",  # 2017-06-02
+        "https://github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz",
     ],
 )
 
diff --git a/tensorflow/tensorboard/BUILD b/tensorflow/tensorboard/BUILD
index 1eb5b12415..caaf1769c0 100644
--- a/tensorflow/tensorboard/BUILD
+++ b/tensorflow/tensorboard/BUILD
@@ -33,6 +33,17 @@ filegroup(
     ],
 )
 
+filegroup(
+    name = "ts_web_library_default_typings",
+    srcs = [
+        # Ordering probably matters.
+        "@com_microsoft_typescript//:lib.es6.d.ts",
+        "@io_angular_clutz//:src/resources/closure.lib.d.ts",
+        "clutz.d.ts",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/tensorboard/clutz.d.ts b/tensorflow/tensorboard/clutz.d.ts
new file mode 100644
index 0000000000..47cf307d26
--- /dev/null
+++ b/tensorflow/tensorboard/clutz.d.ts
@@ -0,0 +1,19 @@
+// Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// tslint:disable
+declare namespace ಠ_ಠ.clutz {
+  interface IteratorIterable<T> extends Iterator<T>, Iterable<T> {}
+  interface IIterableResult<T> extends IteratorResult<T> {}
+}
diff --git a/tensorflow/tensorboard/defs.bzl b/tensorflow/tensorboard/defs.bzl
index b3712a8156..94e2d7c540 100644
--- a/tensorflow/tensorboard/defs.bzl
+++ b/tensorflow/tensorboard/defs.bzl
@@ -15,3 +15,10 @@
 def tensorboard_webcomponent_library(**kwargs):
   """Rules referencing this will be deleted from the codebase soon."""
   pass
+
+def _legacy_js_impl(target, ctx):
+  return struct()
+
+legacy_js = aspect(
+    implementation=_legacy_js_impl,
+    attr_aspects=["exports"])
diff --git a/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java b/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java
index 2635f9b12f..533907dd64 100644
--- a/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java
+++ b/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java
@@ -24,7 +24,6 @@ import com.google.common.base.Joiner;
 import com.google.common.base.Optional;
 import com.google.common.base.Splitter;
 import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
@@ -38,6 +37,7 @@ import com.google.javascript.jscomp.DiagnosticGroup;
 import com.google.javascript.jscomp.DiagnosticGroups;
 import com.google.javascript.jscomp.DiagnosticType;
 import com.google.javascript.jscomp.JSError;
+import com.google.javascript.jscomp.ModuleIdentifier;
 import com.google.javascript.jscomp.PropertyRenamingPolicy;
 import com.google.javascript.jscomp.Result;
 import com.google.javascript.jscomp.SourceFile;
@@ -53,8 +53,10 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -63,6 +65,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Attribute;
 import org.jsoup.nodes.Comment;
@@ -92,6 +95,7 @@ public final class Vulcanize {
   private static final Set<String> legalese = new HashSet<>();
   private static final List<String> licenses = new ArrayList<>();
   private static final List<Webpath> stack = new ArrayList<>();
+  private static final List<SourceFile> externs = new ArrayList<>();
   private static final List<SourceFile> sourcesFromJsLibraries = new ArrayList<>();
   private static final Map<Webpath, String> sourcesFromScriptTags = new LinkedHashMap<>();
   private static final Map<Webpath, Node> sourceTags = new LinkedHashMap<>();
@@ -111,7 +115,13 @@ public final class Vulcanize {
     Path output = Paths.get(args[4]);
     for (int i = 5; i < args.length; i++) {
       if (args[i].endsWith(".js")) {
-        sourcesFromJsLibraries.add(SourceFile.fromFile(args[i]));
+        String code = new String(Files.readAllBytes(Paths.get(args[i])), UTF_8);
+        SourceFile sourceFile = SourceFile.fromCode(args[i], code);
+        if (code.contains("@externs")) {
+          externs.add(sourceFile);
+        } else {
+          sourcesFromJsLibraries.add(sourceFile);
+        }
         continue;
       }
       if (!args[i].endsWith(".pbtxt")) {
@@ -339,12 +349,20 @@ public final class Vulcanize {
     options.setRemoveUnusedPrototypePropertiesInExterns(false);
     options.setRemoveUnusedClassProperties(false);
 
-    // Closure pass.
+    // Dependency management.
     options.setClosurePass(true);
     options.setManageClosureDependencies(true);
     options.getDependencyOptions().setDependencyPruning(true);
-    options.getDependencyOptions().setDependencySorting(false);
+    options.getDependencyOptions().setDependencySorting(true);
     options.getDependencyOptions().setMoocherDropping(false);
+    options.getDependencyOptions()
+        .setEntryPoints(
+            sourceTags
+                .keySet()
+                .stream()
+                .map(Webpath::toString)
+                .map(ModuleIdentifier::forFile)
+                .collect(Collectors.toList()));
 
     // Polymer pass.
     options.setPolymerVersion(1);
@@ -362,6 +380,16 @@ public final class Vulcanize {
         new WarningsGuard() {
           @Override
           public CheckLevel level(JSError error) {
+            if (error.sourceName == null) {
+              return null;
+            }
+            if (error.sourceName.startsWith("javascript/externs")
+                || error.sourceName.contains("com_google_javascript_closure_compiler_externs")) {
+              // TODO(jart): Figure out why these "mismatch of the removeEventListener property on
+              //             type" warnings are showing up.
+              //             https://github.com/google/closure-compiler/pull/1959
+              return CheckLevel.OFF;
+            }
             if (IGNORE_PATHS_PATTERN.matcher(error.sourceName).matches()) {
               return CheckLevel.OFF;
             }
@@ -395,42 +423,39 @@ public final class Vulcanize {
     // Compile everything into a single script.
     Compiler compiler = new Compiler();
     compiler.disableThreads();
-    Result result = compiler.compile(ImmutableList.<SourceFile>of(), sauce, options);
+    Result result = compiler.compile(externs, sauce, options);
     if (!result.success) {
       System.exit(1);
     }
     String jsBlob = compiler.toSource();
 
     // Split apart the JS blob and put it back in the original <script> locations.
+    Deque<Map.Entry<Webpath, Node>> tags = new ArrayDeque<>();
+    tags.addAll(sourceTags.entrySet());
     Matcher matcher = WEBPATH_PATTERN.matcher(jsBlob);
-    Webpath path = null;
-    String pureJsDeps = "";
-    int start = -1;
+    verify(matcher.find(), "Nothing found in compiled JS blob!");
+    Webpath path = Webpath.get(matcher.group(1));
+    int start = 0;
     while (matcher.find()) {
-      if (!sourceTags.containsKey(Webpath.get(matcher.group(1)))) {
-        continue;  // Skip over js_library dependencies, which must group at beginning of args.
-      }
-      if (path != null) {
-        swapScript(path, pureJsDeps + jsBlob.substring(start, matcher.start()));
-        pureJsDeps = "";
-      } else {
-        pureJsDeps = jsBlob.substring(0, matcher.start());
+      if (sourceTags.containsKey(path)) {
+        swapScript(tags, path, jsBlob.substring(start, matcher.start()));
+        start = matcher.start();
       }
       path = Webpath.get(matcher.group(1));
-      start = matcher.start();
-    }
-    swapScript(path, pureJsDeps + jsBlob.substring(start));
-    if (!sourceTags.isEmpty()) {
-      throw new RuntimeException("Couldn't pull out: " + ImmutableSet.copyOf(sourceTags.keySet()));
     }
+    swapScript(tags, path, jsBlob.substring(start));
+    verify(tags.isEmpty(), "<script> wasn't compiled: %s", tags);
   }
 
-  private static void swapScript(Webpath path, String script) {
-    Node tag = sourceTags.get(path);
+  private static void swapScript(
+      Deque<Map.Entry<Webpath, Node>> tags, Webpath path, String script) {
+    verify(!tags.isEmpty(), "jscomp compiled %s after last <script>?!", path);
+    Webpath want = tags.getFirst().getKey();
+    verify(path.equals(want), "<script> tag for %s should come before %s", path, want);
+    Node tag = tags.removeFirst().getValue();
     tag.replaceWith(
         new Element(Tag.valueOf("script"), tag.baseUri())
             .appendChild(new DataNode(script, tag.baseUri())));
-    sourceTags.remove(path);
   }
 
   private static void handleLicense(String text) {
diff --git a/tensorflow/tensorboard/vulcanize.bzl b/tensorflow/tensorboard/vulcanize.bzl
index c82b8cafdb..3aad596328 100644
--- a/tensorflow/tensorboard/vulcanize.bzl
+++ b/tensorflow/tensorboard/vulcanize.bzl
@@ -12,26 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("@io_bazel_rules_closure//closure/private:defs.bzl", "unfurl", "long_path")
+load("//tensorflow/tensorboard:defs.bzl", "legacy_js")
+load("@io_bazel_rules_closure//closure/private:defs.bzl", "collect_js", "unfurl", "long_path")
 load("//tensorflow/tensorboard:web.bzl", "web_aspect")
 
 def _tensorboard_html_binary(ctx):
   deps = unfurl(ctx.attr.deps, provider="webfiles")
-  manifests = depset(order="topological")
-  files = depset()
-  jslibs = depset(ctx.files._jslibs)
-  webpaths = depset()
+  manifests = set(order="topological")
+  files = set()
+  webpaths = set()
   for dep in deps:
     manifests += dep.webfiles.manifests
     webpaths += dep.webfiles.webpaths
     files += dep.data_runfiles.files
-    if hasattr(dep.webfiles, "jslibs"):
-      jslibs += dep.webfiles.jslibs
-    if hasattr(dep, "closure_js_library"):
-      jslibs += getattr(dep.closure_js_library, "srcs", [])
   webpaths += [ctx.attr.output_path]
+  closure_js_library=collect_js(
+      ctx, unfurl(ctx.attr.deps, provider="closure_js_library"))
 
   # vulcanize
+  jslibs = depset(ctx.files._jslibs) + closure_js_library.srcs
   ctx.action(
       inputs=list(manifests | files | jslibs),
       outputs=[ctx.outputs.html],
@@ -101,7 +100,12 @@ tensorboard_html_binary = rule(
         "input_path": attr.string(mandatory=True),
         "output_path": attr.string(mandatory=True),
         "data": attr.label_list(cfg="data", allow_files=True),
-        "deps": attr.label_list(aspects=[web_aspect], mandatory=True),
+        "deps": attr.label_list(
+            aspects=[
+                web_aspect,
+                legacy_js,
+            ],
+            mandatory=True),
         "external_assets": attr.string_dict(default={"/_/runfiles": "."}),
         "_jslibs": attr.label(
             default=Label("//tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize:jslibs"),
diff --git a/tensorflow/tensorboard/web.bzl b/tensorflow/tensorboard/web.bzl
index d3c585a1fd..28fb506fec 100644
--- a/tensorflow/tensorboard/web.bzl
+++ b/tensorflow/tensorboard/web.bzl
@@ -14,7 +14,18 @@
 
 """Same as web_library but supports TypeScript."""
 
+load("//tensorflow/tensorboard:defs.bzl", "legacy_js")
+
+load("//third_party:clutz.bzl",
+     "CLUTZ_ATTRIBUTES",
+     "CLUTZ_OUTPUTS",
+     "clutz_aspect",
+     "extract_dts_from_closure_libraries")
+
 load("@io_bazel_rules_closure//closure/private:defs.bzl",
+     "CLOSURE_LIBRARY_BASE_ATTR",
+     "CLOSURE_LIBRARY_DEPS_ATTR",
+     "collect_js",
      "collect_runfiles",
      "convert_path_to_es6_module_name",
      "create_argfile",
@@ -22,6 +33,14 @@ load("@io_bazel_rules_closure//closure/private:defs.bzl",
      "long_path",
      "unfurl")
 
+_ASPECT_SLURP_FILE_TYPE = FileType([
+    ".html", ".js", ".css", ".gss", ".png", ".jpg", ".gif", ".ico", ".svg"])
+
+_CLOSURE_WORKER = attr.label(
+    default=Label("@io_bazel_rules_closure//java/io/bazel/rules/closure:ClosureWorker"),
+    executable=True,
+    cfg="host")
+
 def _ts_web_library(ctx):
   if not ctx.attr.srcs:
     if ctx.attr.deps:
@@ -43,36 +62,33 @@ def _ts_web_library(ctx):
   # process what came before
   deps = unfurl(ctx.attr.deps, provider="webfiles")
   webpaths = depset()
-  manifests = depset(order="topological")
-  jslibs = depset(order="postorder")
-  ts_typings = depset(ctx.files._es6dts)
-  ts_typings_paths = depset()
+  ts_typings = depset(ctx.files._default_typings)
+  ts_typings_paths = depset(
+      [long_path(ctx, f) for f in ctx.files._default_typings])
   ts_typings_execroots = depset()
+  aspect_runfiles = depset()
   for dep in deps:
     webpaths += dep.webfiles.webpaths
-    manifests += dep.webfiles.manifests
     if hasattr(dep.webfiles, "ts_typings"):
       ts_typings += dep.webfiles.ts_typings
     if hasattr(dep.webfiles, "ts_typings_paths"):
       ts_typings_paths += dep.webfiles.ts_typings_paths
     if hasattr(dep.webfiles, "ts_typings_execroots"):
       ts_typings_execroots += dep.webfiles.ts_typings_execroots
-    if hasattr(dep.webfiles, "jslibs"):
-      jslibs += dep.webfiles.jslibs
-    if hasattr(dep, "closure_js_library"):
-      jslibs += getattr(dep.closure_js_library, "srcs", [])
+    if hasattr(dep.webfiles, "aspect_runfiles"):
+      aspect_runfiles += dep.webfiles.aspect_runfiles
 
   # process what comes now
   manifest_srcs = []
   new_webpaths = []
   ts_inputs = depset()
   ts_outputs = []
-  ts_files = ["lib.es6.d.ts"] + list(ts_typings_paths)
+  ts_files = list(ts_typings_paths)
   new_typings = []
   new_typings_paths = []
   new_typings_execroot = struct(inputs=[])
   execroot = struct(
-      inputs=[("lib.es6.d.ts", ctx.files._es6dts[0].path)],
+      inputs=[(long_path(ctx, f), f.path) for f in ctx.files._default_typings],
       outputs=[],
       program=[ctx.executable._tsc.path, "-p"])
   web_srcs = []
@@ -118,23 +134,20 @@ def _ts_web_library(ctx):
     else:
       web_srcs.append(src)
 
-  # create webfiles manifest
-  manifest = ctx.new_file(ctx.configuration.bin_dir,
-                          "%s.pbtxt" % ctx.label.name)
-  ctx.file_action(
-      output=manifest,
-      content=struct(
-          label=str(ctx.label),
-          src=manifest_srcs).to_proto())
-  manifests += [manifest]
-  webpaths += new_webpaths
+  # get typings for closure code
+  clutz_dts = extract_dts_from_closure_libraries(ctx)
+  if clutz_dts:
+    entry = (long_path(ctx, clutz_dts), clutz_dts.path)
+    ts_inputs += [clutz_dts]
+    ts_files.append(entry[0])
+    execroot.inputs.append(entry)
 
   # compile typescript
   workspace = ""
   if ctx.label.workspace_root:
     workspace = "/" + ctx.label.workspace_root
   if execroot.outputs:
-    ts_config = ctx.new_file(ctx.bin_dir, "%s-tsc.json" % ctx.label.name)
+    ts_config = _new_file(ctx, "-tsc.json")
     execroot.inputs.append(("tsconfig.json", ts_config.path))
     ctx.file_action(
         output=ts_config,
@@ -149,10 +162,9 @@ def _ts_web_library(ctx):
                 noResolve=True,
                 target="es5",
             ),
-            files=list(ts_files),
+            files=ts_files,
         ).to_json())
-    er_config = ctx.new_file(ctx.bin_dir,
-                             "%s-tsc-execroot.json" % ctx.label.name)
+    er_config = _new_file(ctx, "-tsc-execroot.json")
     ctx.file_action(output=er_config, content=execroot.to_json())
     ts_inputs += collect_runfiles([ctx.attr._tsc])
     ts_inputs += ctx.files._tsc
@@ -164,41 +176,14 @@ def _ts_web_library(ctx):
         outputs=ts_outputs,
         executable=ctx.executable._execrooter,
         arguments=[er_config.path] + [f.path for f in ts_typings_execroots],
-        progress_message="Compiling %d TypeScript files" % len(ts_files))
+        progress_message="Compiling %d TypeScript files %s" % (
+            len(ts_files), ctx.label))
 
   # perform strict dependency checking
-  inputs = [manifest]
-  direct_manifests = depset([manifest])
-  args = ["WebfilesValidator",
-          "--dummy", ctx.outputs.dummy.path,
-          "--target", manifest.path]
-  for category in ctx.attr.suppress:
-    args.append("--suppress")
-    args.append(category)
-  inputs.extend(web_srcs)
-  for dep in deps:
-    inputs.append(dep.webfiles.dummy)
-    for f in dep.files:
-      inputs.append(f)
-    direct_manifests += [dep.webfiles.manifest]
-    inputs.append(dep.webfiles.manifest)
-    args.append("--direct_dep")
-    args.append(dep.webfiles.manifest.path)
-  for man in difference(manifests, direct_manifests):
-    inputs.append(man)
-    args.append("--transitive_dep")
-    args.append(man.path)
-  argfile = create_argfile(ctx, args)
-  inputs.append(argfile)
-  ctx.action(
-      inputs=inputs,
-      outputs=[ctx.outputs.dummy],
-      executable=ctx.executable._ClosureWorker,
-      arguments=["@@" + argfile.path],
-      mnemonic="Closure",
-      execution_requirements={"supports-workers": "1"},
-      progress_message="Checking webfiles in %s" % ctx.label)
-  web_srcs.append(ctx.outputs.dummy)
+  manifest = _make_manifest(ctx, manifest_srcs)
+  webpaths += new_webpaths
+  dummy, manifests = _run_webfiles_validator(ctx, web_srcs, deps, manifest)
+  web_srcs.append(dummy)
 
   # define development web server that only applies to this transitive closure
   params = struct(
@@ -207,7 +192,7 @@ def _ts_web_library(ctx):
       manifest=[long_path(ctx, man) for man in manifests],
       external_asset=[struct(webpath=k, path=v)
                       for k, v in ctx.attr.external_assets.items()])
-  params_file = ctx.new_file(ctx.bin_dir, "%s_params.pbtxt" % ctx.label.name)
+  params_file = _new_file(ctx, "-params.pbtxt")
   ctx.file_action(output=params_file, content=params.to_proto())
   ctx.file_action(
       executable=True,
@@ -217,8 +202,7 @@ def _ts_web_library(ctx):
           long_path(ctx, params_file)))
 
   if new_typings:
-    er_config = ctx.new_file(ctx.bin_dir,
-                             "%s-typings-execroot.json" % ctx.label.name)
+    er_config = _new_file(ctx, "-typings-execroot.json")
     ctx.file_action(output=er_config, content=new_typings_execroot.to_json())
     ts_typings += new_typings
     ts_typings_paths += new_typings_paths
@@ -230,83 +214,120 @@ def _ts_web_library(ctx):
 
   # export data to parent rules
   return struct(
-      files=depset(web_srcs),
+      files=depset(web_srcs + [dummy]),
       exports=unfurl(ctx.attr.exports),
       webfiles=struct(
           manifest=manifest,
           manifests=manifests,
           webpaths=webpaths,
-          dummy=ctx.outputs.dummy,
-          jslibs=jslibs,
+          dummy=dummy,
           ts_typings=ts_typings,
           ts_typings_paths=ts_typings_paths,
           ts_typings_execroots=ts_typings_execroots),
+      closure_js_library=collect_js(
+          ctx, unfurl(ctx.attr.deps, provider="closure_js_library")),
       runfiles=ctx.runfiles(
           files=ctx.files.srcs + ctx.files.data + ts_outputs + [
               manifest,
               params_file,
               ctx.outputs.executable,
-              ctx.outputs.dummy],
+              dummy],
           transitive_files=(collect_runfiles([ctx.attr._WebfilesServer]) |
                             collect_runfiles(deps) |
-                            collect_runfiles(ctx.attr.data))))
+                            collect_runfiles(ctx.attr.data) |
+                            aspect_runfiles)))
 
 def _web_aspect_impl(target, ctx):
-  if ctx.rule.kind in ("js_library", "pinto_library"):
-    return _web_aspect_js_library(target, ctx, [], depset())
-  if hasattr(target, "js"):
-    return _web_aspect_js_library(
-        target,
-        ctx,
-        target.files,
-        target.js.full_tc(True))
-  return struct()
-
-def _web_aspect_js_library(target, ctx, extra_srcs, extra_transitive):
-  deps = unfurl((ctx.rule.attr.deps +
-                 getattr(ctx.rule.attr, 'sticky_deps', [])),
-                provider="webfiles")
-  # process what came before
+  if hasattr(target, "webfiles"):
+    return struct()
+  srcs = []
+  deps = []
+  if hasattr(ctx.rule.files, "srcs"):
+    srcs.extend(_ASPECT_SLURP_FILE_TYPE.filter(ctx.rule.files.srcs))
+  for attr in ("deps", "sticky_deps", "module_deps"):
+    value = getattr(ctx.rule.attr, attr, None)
+    if value:
+      deps.extend(value)
+  deps = unfurl(deps, provider="webfiles")
   webpaths = depset()
-  manifests = depset(order="topological")
-  jslibs = depset(order="postorder")
+  aspect_runfiles = depset(srcs)
   for dep in deps:
     webpaths += dep.webfiles.webpaths
-    manifests += dep.webfiles.manifests
-    if hasattr(dep.webfiles, "jslibs"):
-      jslibs += dep.webfiles.jslibs
-  # process what comes now
-  srcs = ctx.rule.files.srcs + extra_srcs
-  jslibs += [src for src in srcs if src.path.endswith(".js")]
+    if hasattr(dep.webfiles, "aspect_runfiles"):
+      aspect_runfiles += dep.webfiles.aspect_runfiles
   manifest_srcs = []
   new_webpaths = []
-  web_srcs = []
   for src in srcs:
     webpath = "/" + long_path(ctx, src)
     _add_webpath(ctx, src, webpath, webpaths, new_webpaths, manifest_srcs)
-    web_srcs.append(src)
-  # create webfiles manifest
-  manifest = ctx.new_file(ctx.configuration.bin_dir,
-                          "%s-webfiles.pbtxt" % ctx.label.name)
-  ctx.file_action(
-      output=manifest,
-      content=struct(
-          label=str(ctx.label),
-          src=manifest_srcs).to_proto())
-  manifests += [manifest]
   webpaths += new_webpaths
+  manifest = _make_manifest(ctx, manifest_srcs)
+  dummy, manifests = _run_webfiles_validator(ctx, srcs, deps, manifest)
+  aspect_runfiles += [dummy, manifest]
   return struct(
-      exports=[] if srcs else deps,
       webfiles=struct(
           manifest=manifest,
           manifests=manifests,
           webpaths=webpaths,
-          dummy=manifest,
-          jslibs=jslibs),
-      closure_legacy_js_runfiles=(depset(srcs + ctx.rule.files.data) |
-                                  extra_transitive |
-                                  collect_runfiles(deps) |
-                                  collect_runfiles(ctx.rule.files.data)))
+          dummy=dummy,
+          aspect_runfiles=aspect_runfiles))
+
+def _make_manifest(ctx, src_list):
+  manifest = _new_file(ctx, "-webfiles.pbtxt")
+  ctx.file_action(
+      output=manifest,
+      content=struct(
+          label=str(ctx.label),
+          src=src_list).to_proto())
+  return manifest
+
+def _run_webfiles_validator(ctx, srcs, deps, manifest):
+  dummy = _new_file(ctx, "-webfiles.ignoreme")
+  manifests = depset(order="topological")
+  for dep in deps:
+    manifests += dep.webfiles.manifests
+  if srcs:
+    args = ["WebfilesValidator",
+            "--dummy", dummy.path,
+            "--target", manifest.path]
+    if hasattr(ctx, "attr") and hasattr(ctx.attr, "suppress"):
+      for category in ctx.attr.suppress:
+        args.append("--suppress")
+        args.append(category)
+    inputs = [manifest]
+    inputs.extend(srcs)
+    direct_manifests = depset()
+    for dep in deps:
+      inputs.append(dep.webfiles.dummy)
+      for f in dep.files:
+        inputs.append(f)
+      direct_manifests += [dep.webfiles.manifest]
+      inputs.append(dep.webfiles.manifest)
+      args.append("--direct_dep")
+      args.append(dep.webfiles.manifest.path)
+    for man in difference(manifests, direct_manifests):
+      inputs.append(man)
+      args.append("--transitive_dep")
+      args.append(man.path)
+    argfile = _new_file(ctx, "-webfiles-checker-args.txt")
+    ctx.file_action(output=argfile, content="\n".join(args))
+    inputs.append(argfile)
+    ctx.action(
+        inputs=inputs,
+        outputs=[dummy],
+        executable=(getattr(ctx.executable, "_ClosureWorker", None) or
+                    getattr(ctx.executable, "_ClosureWorkerAspect", None)),
+        arguments=["@@" + argfile.path],
+        mnemonic="Closure",
+        execution_requirements={"supports-workers": "1"},
+        progress_message="Checking webfiles %s" % ctx.label)
+  else:
+    ctx.file_action(output=dummy, content="BOO!")
+  manifests += [manifest]
+  return dummy, manifests
+
+def _new_file(ctx, suffix):
+  return ctx.new_file(ctx.bin_dir, "%s%s" % (ctx.label.name, suffix))
 
 def _add_webpath(ctx, src, webpath, webpaths, new_webpaths, manifest_srcs):
   if webpath in new_webpaths:
@@ -354,45 +375,45 @@ def _get_strip(ctx):
 
 web_aspect = aspect(
     implementation=_web_aspect_impl,
-    attr_aspects=["deps"])
+    attr_aspects=["deps", "sticky_deps", "module_deps"],
+    attrs={"_ClosureWorkerAspect": _CLOSURE_WORKER})
 
 ts_web_library = rule(
     implementation=_ts_web_library,
     executable=True,
-    attrs={
+    attrs=CLUTZ_ATTRIBUTES + {
         "path": attr.string(),
         "srcs": attr.label_list(allow_files=True),
-        "deps": attr.label_list(aspects=[web_aspect]),
+        "deps": attr.label_list(
+            aspects=[
+                web_aspect,
+                clutz_aspect,
+                legacy_js,
+            ]),
         "exports": attr.label_list(),
         "data": attr.label_list(cfg="data", allow_files=True),
         "suppress": attr.string_list(),
         "strip_prefix": attr.string(),
         "external_assets": attr.string_dict(default={"/_/runfiles": "."}),
+        "clutz_entry_points": attr.string_list(),
         "_execrooter": attr.label(
-            default=Label(
-                "//tensorflow/tensorboard/scripts:execrooter"),
+            default=Label("//tensorflow/tensorboard/scripts:execrooter"),
             executable=True,
             cfg="host"),
         "_tsc": attr.label(
-            default=Label(
-                "@com_microsoft_typescript//:tsc"),
+            default=Label("@com_microsoft_typescript//:tsc"),
             allow_files=True,
             executable=True,
             cfg="host"),
-        "_es6dts": attr.label(
-            default=Label(
-                "@com_microsoft_typescript//:lib.es6.d.ts"),
+        "_default_typings": attr.label(
+            default=Label("//tensorflow/tensorboard:ts_web_library_default_typings"),
             allow_files=True),
-        "_ClosureWorker": attr.label(
-            default=Label("@io_bazel_rules_closure//java/io/bazel/rules/closure:ClosureWorker"),
-            executable=True,
-            cfg="host"),
         "_WebfilesServer": attr.label(
-            default=Label(
-                "@io_bazel_rules_closure//java/io/bazel/rules/closure/webfiles/server:WebfilesServer"),
+            default=Label("@io_bazel_rules_closure//java/io/bazel/rules/closure/webfiles/server:WebfilesServer"),
             executable=True,
             cfg="host"),
+        "_ClosureWorker": _CLOSURE_WORKER,
+        "_closure_library_base": CLOSURE_LIBRARY_BASE_ATTR,
+        "_closure_library_deps": CLOSURE_LIBRARY_DEPS_ATTR,
     },
-    outputs={
-        "dummy": "%{name}.ignoreme",
-    })
+    outputs=CLUTZ_OUTPUTS)
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 18bc9a8277..1abb3abfc0 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -741,19 +741,27 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ]),
   )
 
-  ##############################################################################
-  # TensorBoard JavaScript Production Dependencies
+  native.new_http_archive(
+      name = "io_angular_clutz",
+      build_file = "//third_party:clutz.BUILD",
+      sha256 = "2981de41d1ff4774b544423da9a2cd8beb3be649e95aef2ef2fd83957300b3fe",
+      strip_prefix = "clutz-b0db5ade9bb535d387f05292316c422790c9848e",
+      urls = [
+          "http://mirror.bazel.build/github.com/angular/clutz/archive/b0db5ade9bb535d387f05292316c422790c9848e.tar.gz",  # 2017-05-22
+          "https://github.com/angular/clutz/archive/b0db5ade9bb535d387f05292316c422790c9848e.tar.gz",
+      ],
+  )
 
   filegroup_external(
       name = "com_google_javascript_closure_compiler_externs",
       licenses = ["notice"],  # Apache 2.0
       sha256_urls_extract = {
-          "0ee7b88ed2955b622eaa038bece283e28d0fb5abebfbb80871fc3d0353f0000b": [
-              "http://mirror.bazel.build/github.com/google/closure-compiler/archive/v20170423.tar.gz",
-              "https://github.com/google/closure-compiler/archive/v20170423.tar.gz",
+          "0f515a6ebfa138490b3c5ea9f3591ea1a7e4a930d3074f18b3eca86084ad9b66": [
+              "http://mirror.bazel.build/github.com/google/closure-compiler/archive/b37e6000001b0a6bf4c0be49024ebda14a8711d9.tar.gz",  # 2017-06-02
+              "https://github.com/google/closure-compiler/archive/b37e6000001b0a6bf4c0be49024ebda14a8711d9.tar.gz",
           ],
       },
-      strip_prefix = {"v20170423.tar.gz": "closure-compiler-20170423/externs"},
+      strip_prefix = {"b37e6000001b0a6bf4c0be49024ebda14a8711d9.tar.gz": "closure-compiler-b37e6000001b0a6bf4c0be49024ebda14a8711d9/externs"},
   )
 
   filegroup_external(
@@ -761,12 +769,15 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       licenses = ["notice"],  # Apache 2.0
       sha256_urls = {
           "23baad9a200a717a821c6df504c84d3a893d7ea9102b14876eb80097e3b94292": [
-              "http://mirror.bazel.build/raw.githubusercontent.com/google/closure-compiler/0e8dc5597a295ee259e3fecd98d6535dc621232f/contrib/externs/polymer-1.0.js",
+              "http://mirror.bazel.build/raw.githubusercontent.com/google/closure-compiler/0e8dc5597a295ee259e3fecd98d6535dc621232f/contrib/externs/polymer-1.0.js",  # 2017-05-27
               "https://raw.githubusercontent.com/google/closure-compiler/0e8dc5597a295ee259e3fecd98d6535dc621232f/contrib/externs/polymer-1.0.js",
           ],
       },
   )
 
+  ##############################################################################
+  # TensorBoard JavaScript Production Dependencies
+
   web_library_external(
       name = "com_lodash",
       licenses = ["notice"],  # MIT
diff --git a/third_party/clutz.BUILD b/third_party/clutz.BUILD
new file mode 100644
index 0000000000..593b70366a
--- /dev/null
+++ b/third_party/clutz.BUILD
@@ -0,0 +1,44 @@
+# Description:
+#   Build tool for making TypeScript .d.ts files from Closure JavaScript.
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # MIT
+
+exports_files([
+    "LICENSE",
+    "src/resources/closure.lib.d.ts",
+])
+
+JVM_FLAGS = [
+    "-Xss20m",  # JSCompiler needs big stacks for recursive parsing
+    "-XX:+UseParallelGC",  # Best GC when app isn't latency sensitive
+]
+
+java_binary(
+    name = "clutz",
+    srcs = glob(["src/main/java/com/google/javascript/clutz/**/*.java"]),
+    jvm_flags = JVM_FLAGS,
+    main_class = "com.google.javascript.clutz.DeclarationGenerator",
+    deps = [
+        "@args4j",
+        "@com_google_code_findbugs_jsr305",
+        "@com_google_code_gson",
+        "@com_google_guava",
+        "@com_google_javascript_closure_compiler",
+    ],
+)
+
+java_binary(
+    name = "gents",
+    srcs = glob(["src/main/java/com/google/javascript/gents/**/*.java"]),
+    jvm_flags = JVM_FLAGS,
+    main_class = "com.google.javascript.gents.TypeScriptGenerator",
+    deps = [
+        "@args4j",
+        "@com_google_code_findbugs_jsr305",
+        "@com_google_code_gson",
+        "@com_google_guava",
+        "@com_google_javascript_closure_compiler",
+    ],
+)
diff --git a/third_party/clutz.bzl b/third_party/clutz.bzl
new file mode 100644
index 0000000000..f273c78c79
--- /dev/null
+++ b/third_party/clutz.bzl
@@ -0,0 +1,77 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Build definitions for TypeScript from Closure JavaScript libraries."""
+
+load("@io_bazel_rules_closure//closure/private:defs.bzl",
+     "JS_FILE_TYPE",
+     "collect_js",
+     "unfurl")
+
+CLUTZ_ATTRIBUTES = {
+    "_clutz": attr.label(
+        default=Label("@io_angular_clutz//:clutz"),
+        executable=True,
+        cfg="host"),
+    "_clutz_externs": attr.label(
+        default=Label("@com_google_javascript_closure_compiler_externs"),
+        allow_files=True),
+}
+
+def extract_dts_from_closure_libraries(ctx):
+  """Extracts type definitions from closure dependencies.
+
+  This just generates one big .d.ts file for all transitive Closure sources,
+  and does not pass it down. That means each rule has to duplicate the effort,
+  but on the other hand allows transitive dependencies on shared rules without
+  causing duplicate definition errors.
+
+  Args:
+      ctx: A Skylark context.
+  Returns:
+      The generated Clutz typings file, or None if there were no JS deps.
+  """
+  deps = unfurl(ctx.attr.deps, provider="closure_js_library")
+  js = collect_js(ctx, deps)
+  if not js.srcs:
+    return None
+  js_typings = ctx.new_file(ctx.bin_dir, "%s-js-typings.d.ts" % ctx.label.name)
+  srcs = depset(JS_FILE_TYPE.filter(ctx.files._clutz_externs)) + js.srcs
+  args = ["-o", js_typings.path]
+  for src in srcs:
+    args.append(src.path)
+  if getattr(ctx.attr, "clutz_entry_points", None):
+    args.append("--closure_entry_points")
+    args.extend(ctx.attr.clutz_entry_points)
+  ctx.action(
+      inputs=list(srcs),
+      outputs=[js_typings],
+      executable=ctx.executable._clutz,
+      arguments=args,
+      mnemonic="Clutz",
+      progress_message="Running Clutz on %d JS files %s" % (
+          len(srcs), ctx.label))
+  return js_typings
+
+################################################################################
+# The following definitions are for API compatibility with internal clutz.bzl
+
+CLUTZ_OUTPUTS = {}
+
+def _clutz_aspect_impl(target, ctx):
+  return struct()
+
+clutz_aspect = aspect(
+    implementation=_clutz_aspect_impl,
+    attr_aspects=["exports"])
-- 
cgit v1.2.3