diff options
-rw-r--r-- | WORKSPACE | 8 | ||||
-rw-r--r-- | tensorflow/tensorboard/BUILD | 11 | ||||
-rw-r--r-- | tensorflow/tensorboard/clutz.d.ts | 19 | ||||
-rw-r--r-- | tensorflow/tensorboard/defs.bzl | 7 | ||||
-rw-r--r-- | tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java | 73 | ||||
-rw-r--r-- | tensorflow/tensorboard/vulcanize.bzl | 24 | ||||
-rw-r--r-- | tensorflow/tensorboard/web.bzl | 273 | ||||
-rw-r--r-- | tensorflow/workspace.bzl | 25 | ||||
-rw-r--r-- | third_party/clutz.BUILD | 44 | ||||
-rw-r--r-- | third_party/clutz.bzl | 77 |
10 files changed, 390 insertions, 171 deletions
@@ -2,11 +2,11 @@ workspace(name = "org_tensorflow") http_archive( name = "io_bazel_rules_closure", - sha256 = "edc91f556b762fc5212d1050d00b12e40dd0b0b1c1d5d96886b59e9a30a6cae4", - strip_prefix = "rules_closure-3f07fb6a58870afbb36051bd5d54da4479561cc6", + sha256 = "bc41b80486413aaa551860fc37471dbc0666e1dbb5236fb6177cb83b0c105846", + strip_prefix = "rules_closure-dec425a4ff3faf09a56c85d082e4eed05d8ce38f", urls = [ - "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/3f07fb6a58870afbb36051bd5d54da4479561cc6.tar.gz", # 2017-05-31 - "https://github.com/bazelbuild/rules_closure/archive/3f07fb6a58870afbb36051bd5d54da4479561cc6.tar.gz", + "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", # 2017-06-02 + "https://github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", ], ) diff --git a/tensorflow/tensorboard/BUILD b/tensorflow/tensorboard/BUILD index 1eb5b12415..caaf1769c0 100644 --- a/tensorflow/tensorboard/BUILD +++ b/tensorflow/tensorboard/BUILD @@ -34,6 +34,17 @@ filegroup( ) filegroup( + name = "ts_web_library_default_typings", + srcs = [ + # Ordering probably matters. + "@com_microsoft_typescript//:lib.es6.d.ts", + "@io_angular_clutz//:src/resources/closure.lib.d.ts", + "clutz.d.ts", + ], + visibility = ["//visibility:public"], +) + +filegroup( name = "all_files", srcs = glob( ["**"], diff --git a/tensorflow/tensorboard/clutz.d.ts b/tensorflow/tensorboard/clutz.d.ts new file mode 100644 index 0000000000..47cf307d26 --- /dev/null +++ b/tensorflow/tensorboard/clutz.d.ts @@ -0,0 +1,19 @@ +// Copyright 2017 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// tslint:disable +declare namespace ಠ_ಠ.clutz { + interface IteratorIterable<T> extends Iterator<T>, Iterable<T> {} + interface IIterableResult<T> extends IteratorResult<T> {} +} diff --git a/tensorflow/tensorboard/defs.bzl b/tensorflow/tensorboard/defs.bzl index b3712a8156..94e2d7c540 100644 --- a/tensorflow/tensorboard/defs.bzl +++ b/tensorflow/tensorboard/defs.bzl @@ -15,3 +15,10 @@ def tensorboard_webcomponent_library(**kwargs): """Rules referencing this will be deleted from the codebase soon.""" pass + +def _legacy_js_impl(target, ctx): + return struct() + +legacy_js = aspect( + implementation=_legacy_js_impl, + attr_aspects=["exports"]) diff --git a/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java b/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java index 2635f9b12f..533907dd64 100644 --- a/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java +++ b/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java @@ -24,7 +24,6 @@ import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.base.Splitter; import com.google.common.collect.HashMultimap; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; @@ -38,6 +37,7 @@ import com.google.javascript.jscomp.DiagnosticGroup; import com.google.javascript.jscomp.DiagnosticGroups; import com.google.javascript.jscomp.DiagnosticType; import com.google.javascript.jscomp.JSError; +import com.google.javascript.jscomp.ModuleIdentifier; import com.google.javascript.jscomp.PropertyRenamingPolicy; import com.google.javascript.jscomp.Result; import com.google.javascript.jscomp.SourceFile; @@ -53,8 +53,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; +import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -63,6 +65,7 @@ import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.jsoup.Jsoup; import org.jsoup.nodes.Attribute; import org.jsoup.nodes.Comment; @@ -92,6 +95,7 @@ public final class Vulcanize { private static final Set<String> legalese = new HashSet<>(); private static final List<String> licenses = new ArrayList<>(); private static final List<Webpath> stack = new ArrayList<>(); + private static final List<SourceFile> externs = new ArrayList<>(); private static final List<SourceFile> sourcesFromJsLibraries = new ArrayList<>(); private static final Map<Webpath, String> sourcesFromScriptTags = new LinkedHashMap<>(); private static final Map<Webpath, Node> sourceTags = new LinkedHashMap<>(); @@ -111,7 +115,13 @@ public final class Vulcanize { Path output = Paths.get(args[4]); for (int i = 5; i < args.length; i++) { if (args[i].endsWith(".js")) { - sourcesFromJsLibraries.add(SourceFile.fromFile(args[i])); + String code = new String(Files.readAllBytes(Paths.get(args[i])), UTF_8); + SourceFile sourceFile = SourceFile.fromCode(args[i], code); + if (code.contains("@externs")) { + externs.add(sourceFile); + } else { + sourcesFromJsLibraries.add(sourceFile); + } continue; } if (!args[i].endsWith(".pbtxt")) { @@ -339,12 +349,20 @@ public final class Vulcanize { options.setRemoveUnusedPrototypePropertiesInExterns(false); options.setRemoveUnusedClassProperties(false); - // Closure pass. + // Dependency management. options.setClosurePass(true); options.setManageClosureDependencies(true); options.getDependencyOptions().setDependencyPruning(true); - options.getDependencyOptions().setDependencySorting(false); + options.getDependencyOptions().setDependencySorting(true); options.getDependencyOptions().setMoocherDropping(false); + options.getDependencyOptions() + .setEntryPoints( + sourceTags + .keySet() + .stream() + .map(Webpath::toString) + .map(ModuleIdentifier::forFile) + .collect(Collectors.toList())); // Polymer pass. options.setPolymerVersion(1); @@ -362,6 +380,16 @@ public final class Vulcanize { new WarningsGuard() { @Override public CheckLevel level(JSError error) { + if (error.sourceName == null) { + return null; + } + if (error.sourceName.startsWith("javascript/externs") + || error.sourceName.contains("com_google_javascript_closure_compiler_externs")) { + // TODO(jart): Figure out why these "mismatch of the removeEventListener property on + // type" warnings are showing up. + // https://github.com/google/closure-compiler/pull/1959 + return CheckLevel.OFF; + } if (IGNORE_PATHS_PATTERN.matcher(error.sourceName).matches()) { return CheckLevel.OFF; } @@ -395,42 +423,39 @@ public final class Vulcanize { // Compile everything into a single script. Compiler compiler = new Compiler(); compiler.disableThreads(); - Result result = compiler.compile(ImmutableList.<SourceFile>of(), sauce, options); + Result result = compiler.compile(externs, sauce, options); if (!result.success) { System.exit(1); } String jsBlob = compiler.toSource(); // Split apart the JS blob and put it back in the original <script> locations. + Deque<Map.Entry<Webpath, Node>> tags = new ArrayDeque<>(); + tags.addAll(sourceTags.entrySet()); Matcher matcher = WEBPATH_PATTERN.matcher(jsBlob); - Webpath path = null; - String pureJsDeps = ""; - int start = -1; + verify(matcher.find(), "Nothing found in compiled JS blob!"); + Webpath path = Webpath.get(matcher.group(1)); + int start = 0; while (matcher.find()) { - if (!sourceTags.containsKey(Webpath.get(matcher.group(1)))) { - continue; // Skip over js_library dependencies, which must group at beginning of args. - } - if (path != null) { - swapScript(path, pureJsDeps + jsBlob.substring(start, matcher.start())); - pureJsDeps = ""; - } else { - pureJsDeps = jsBlob.substring(0, matcher.start()); + if (sourceTags.containsKey(path)) { + swapScript(tags, path, jsBlob.substring(start, matcher.start())); + start = matcher.start(); } path = Webpath.get(matcher.group(1)); - start = matcher.start(); - } - swapScript(path, pureJsDeps + jsBlob.substring(start)); - if (!sourceTags.isEmpty()) { - throw new RuntimeException("Couldn't pull out: " + ImmutableSet.copyOf(sourceTags.keySet())); } + swapScript(tags, path, jsBlob.substring(start)); + verify(tags.isEmpty(), "<script> wasn't compiled: %s", tags); } - private static void swapScript(Webpath path, String script) { - Node tag = sourceTags.get(path); + private static void swapScript( + Deque<Map.Entry<Webpath, Node>> tags, Webpath path, String script) { + verify(!tags.isEmpty(), "jscomp compiled %s after last <script>?!", path); + Webpath want = tags.getFirst().getKey(); + verify(path.equals(want), "<script> tag for %s should come before %s", path, want); + Node tag = tags.removeFirst().getValue(); tag.replaceWith( new Element(Tag.valueOf("script"), tag.baseUri()) .appendChild(new DataNode(script, tag.baseUri()))); - sourceTags.remove(path); } private static void handleLicense(String text) { diff --git a/tensorflow/tensorboard/vulcanize.bzl b/tensorflow/tensorboard/vulcanize.bzl index c82b8cafdb..3aad596328 100644 --- a/tensorflow/tensorboard/vulcanize.bzl +++ b/tensorflow/tensorboard/vulcanize.bzl @@ -12,26 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@io_bazel_rules_closure//closure/private:defs.bzl", "unfurl", "long_path") +load("//tensorflow/tensorboard:defs.bzl", "legacy_js") +load("@io_bazel_rules_closure//closure/private:defs.bzl", "collect_js", "unfurl", "long_path") load("//tensorflow/tensorboard:web.bzl", "web_aspect") def _tensorboard_html_binary(ctx): deps = unfurl(ctx.attr.deps, provider="webfiles") - manifests = depset(order="topological") - files = depset() - jslibs = depset(ctx.files._jslibs) - webpaths = depset() + manifests = set(order="topological") + files = set() + webpaths = set() for dep in deps: manifests += dep.webfiles.manifests webpaths += dep.webfiles.webpaths files += dep.data_runfiles.files - if hasattr(dep.webfiles, "jslibs"): - jslibs += dep.webfiles.jslibs - if hasattr(dep, "closure_js_library"): - jslibs += getattr(dep.closure_js_library, "srcs", []) webpaths += [ctx.attr.output_path] + closure_js_library=collect_js( + ctx, unfurl(ctx.attr.deps, provider="closure_js_library")) # vulcanize + jslibs = depset(ctx.files._jslibs) + closure_js_library.srcs ctx.action( inputs=list(manifests | files | jslibs), outputs=[ctx.outputs.html], @@ -101,7 +100,12 @@ tensorboard_html_binary = rule( "input_path": attr.string(mandatory=True), "output_path": attr.string(mandatory=True), "data": attr.label_list(cfg="data", allow_files=True), - "deps": attr.label_list(aspects=[web_aspect], mandatory=True), + "deps": attr.label_list( + aspects=[ + web_aspect, + legacy_js, + ], + mandatory=True), "external_assets": attr.string_dict(default={"/_/runfiles": "."}), "_jslibs": attr.label( default=Label("//tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize:jslibs"), diff --git a/tensorflow/tensorboard/web.bzl b/tensorflow/tensorboard/web.bzl index d3c585a1fd..28fb506fec 100644 --- a/tensorflow/tensorboard/web.bzl +++ b/tensorflow/tensorboard/web.bzl @@ -14,7 +14,18 @@ """Same as web_library but supports TypeScript.""" +load("//tensorflow/tensorboard:defs.bzl", "legacy_js") + +load("//third_party:clutz.bzl", + "CLUTZ_ATTRIBUTES", + "CLUTZ_OUTPUTS", + "clutz_aspect", + "extract_dts_from_closure_libraries") + load("@io_bazel_rules_closure//closure/private:defs.bzl", + "CLOSURE_LIBRARY_BASE_ATTR", + "CLOSURE_LIBRARY_DEPS_ATTR", + "collect_js", "collect_runfiles", "convert_path_to_es6_module_name", "create_argfile", @@ -22,6 +33,14 @@ load("@io_bazel_rules_closure//closure/private:defs.bzl", "long_path", "unfurl") +_ASPECT_SLURP_FILE_TYPE = FileType([ + ".html", ".js", ".css", ".gss", ".png", ".jpg", ".gif", ".ico", ".svg"]) + +_CLOSURE_WORKER = attr.label( + default=Label("@io_bazel_rules_closure//java/io/bazel/rules/closure:ClosureWorker"), + executable=True, + cfg="host") + def _ts_web_library(ctx): if not ctx.attr.srcs: if ctx.attr.deps: @@ -43,36 +62,33 @@ def _ts_web_library(ctx): # process what came before deps = unfurl(ctx.attr.deps, provider="webfiles") webpaths = depset() - manifests = depset(order="topological") - jslibs = depset(order="postorder") - ts_typings = depset(ctx.files._es6dts) - ts_typings_paths = depset() + ts_typings = depset(ctx.files._default_typings) + ts_typings_paths = depset( + [long_path(ctx, f) for f in ctx.files._default_typings]) ts_typings_execroots = depset() + aspect_runfiles = depset() for dep in deps: webpaths += dep.webfiles.webpaths - manifests += dep.webfiles.manifests if hasattr(dep.webfiles, "ts_typings"): ts_typings += dep.webfiles.ts_typings if hasattr(dep.webfiles, "ts_typings_paths"): ts_typings_paths += dep.webfiles.ts_typings_paths if hasattr(dep.webfiles, "ts_typings_execroots"): ts_typings_execroots += dep.webfiles.ts_typings_execroots - if hasattr(dep.webfiles, "jslibs"): - jslibs += dep.webfiles.jslibs - if hasattr(dep, "closure_js_library"): - jslibs += getattr(dep.closure_js_library, "srcs", []) + if hasattr(dep.webfiles, "aspect_runfiles"): + aspect_runfiles += dep.webfiles.aspect_runfiles # process what comes now manifest_srcs = [] new_webpaths = [] ts_inputs = depset() ts_outputs = [] - ts_files = ["lib.es6.d.ts"] + list(ts_typings_paths) + ts_files = list(ts_typings_paths) new_typings = [] new_typings_paths = [] new_typings_execroot = struct(inputs=[]) execroot = struct( - inputs=[("lib.es6.d.ts", ctx.files._es6dts[0].path)], + inputs=[(long_path(ctx, f), f.path) for f in ctx.files._default_typings], outputs=[], program=[ctx.executable._tsc.path, "-p"]) web_srcs = [] @@ -118,23 +134,20 @@ def _ts_web_library(ctx): else: web_srcs.append(src) - # create webfiles manifest - manifest = ctx.new_file(ctx.configuration.bin_dir, - "%s.pbtxt" % ctx.label.name) - ctx.file_action( - output=manifest, - content=struct( - label=str(ctx.label), - src=manifest_srcs).to_proto()) - manifests += [manifest] - webpaths += new_webpaths + # get typings for closure code + clutz_dts = extract_dts_from_closure_libraries(ctx) + if clutz_dts: + entry = (long_path(ctx, clutz_dts), clutz_dts.path) + ts_inputs += [clutz_dts] + ts_files.append(entry[0]) + execroot.inputs.append(entry) # compile typescript workspace = "" if ctx.label.workspace_root: workspace = "/" + ctx.label.workspace_root if execroot.outputs: - ts_config = ctx.new_file(ctx.bin_dir, "%s-tsc.json" % ctx.label.name) + ts_config = _new_file(ctx, "-tsc.json") execroot.inputs.append(("tsconfig.json", ts_config.path)) ctx.file_action( output=ts_config, @@ -149,10 +162,9 @@ def _ts_web_library(ctx): noResolve=True, target="es5", ), - files=list(ts_files), + files=ts_files, ).to_json()) - er_config = ctx.new_file(ctx.bin_dir, - "%s-tsc-execroot.json" % ctx.label.name) + er_config = _new_file(ctx, "-tsc-execroot.json") ctx.file_action(output=er_config, content=execroot.to_json()) ts_inputs += collect_runfiles([ctx.attr._tsc]) ts_inputs += ctx.files._tsc @@ -164,41 +176,14 @@ def _ts_web_library(ctx): outputs=ts_outputs, executable=ctx.executable._execrooter, arguments=[er_config.path] + [f.path for f in ts_typings_execroots], - progress_message="Compiling %d TypeScript files" % len(ts_files)) + progress_message="Compiling %d TypeScript files %s" % ( + len(ts_files), ctx.label)) # perform strict dependency checking - inputs = [manifest] - direct_manifests = depset([manifest]) - args = ["WebfilesValidator", - "--dummy", ctx.outputs.dummy.path, - "--target", manifest.path] - for category in ctx.attr.suppress: - args.append("--suppress") - args.append(category) - inputs.extend(web_srcs) - for dep in deps: - inputs.append(dep.webfiles.dummy) - for f in dep.files: - inputs.append(f) - direct_manifests += [dep.webfiles.manifest] - inputs.append(dep.webfiles.manifest) - args.append("--direct_dep") - args.append(dep.webfiles.manifest.path) - for man in difference(manifests, direct_manifests): - inputs.append(man) - args.append("--transitive_dep") - args.append(man.path) - argfile = create_argfile(ctx, args) - inputs.append(argfile) - ctx.action( - inputs=inputs, - outputs=[ctx.outputs.dummy], - executable=ctx.executable._ClosureWorker, - arguments=["@@" + argfile.path], - mnemonic="Closure", - execution_requirements={"supports-workers": "1"}, - progress_message="Checking webfiles in %s" % ctx.label) - web_srcs.append(ctx.outputs.dummy) + manifest = _make_manifest(ctx, manifest_srcs) + webpaths += new_webpaths + dummy, manifests = _run_webfiles_validator(ctx, web_srcs, deps, manifest) + web_srcs.append(dummy) # define development web server that only applies to this transitive closure params = struct( @@ -207,7 +192,7 @@ def _ts_web_library(ctx): manifest=[long_path(ctx, man) for man in manifests], external_asset=[struct(webpath=k, path=v) for k, v in ctx.attr.external_assets.items()]) - params_file = ctx.new_file(ctx.bin_dir, "%s_params.pbtxt" % ctx.label.name) + params_file = _new_file(ctx, "-params.pbtxt") ctx.file_action(output=params_file, content=params.to_proto()) ctx.file_action( executable=True, @@ -217,8 +202,7 @@ def _ts_web_library(ctx): long_path(ctx, params_file))) if new_typings: - er_config = ctx.new_file(ctx.bin_dir, - "%s-typings-execroot.json" % ctx.label.name) + er_config = _new_file(ctx, "-typings-execroot.json") ctx.file_action(output=er_config, content=new_typings_execroot.to_json()) ts_typings += new_typings ts_typings_paths += new_typings_paths @@ -230,83 +214,120 @@ def _ts_web_library(ctx): # export data to parent rules return struct( - files=depset(web_srcs), + files=depset(web_srcs + [dummy]), exports=unfurl(ctx.attr.exports), webfiles=struct( manifest=manifest, manifests=manifests, webpaths=webpaths, - dummy=ctx.outputs.dummy, - jslibs=jslibs, + dummy=dummy, ts_typings=ts_typings, ts_typings_paths=ts_typings_paths, ts_typings_execroots=ts_typings_execroots), + closure_js_library=collect_js( + ctx, unfurl(ctx.attr.deps, provider="closure_js_library")), runfiles=ctx.runfiles( files=ctx.files.srcs + ctx.files.data + ts_outputs + [ manifest, params_file, ctx.outputs.executable, - ctx.outputs.dummy], + dummy], transitive_files=(collect_runfiles([ctx.attr._WebfilesServer]) | collect_runfiles(deps) | - collect_runfiles(ctx.attr.data)))) + collect_runfiles(ctx.attr.data) | + aspect_runfiles))) def _web_aspect_impl(target, ctx): - if ctx.rule.kind in ("js_library", "pinto_library"): - return _web_aspect_js_library(target, ctx, [], depset()) - if hasattr(target, "js"): - return _web_aspect_js_library( - target, - ctx, - target.files, - target.js.full_tc(True)) - return struct() - -def _web_aspect_js_library(target, ctx, extra_srcs, extra_transitive): - deps = unfurl((ctx.rule.attr.deps + - getattr(ctx.rule.attr, 'sticky_deps', [])), - provider="webfiles") - # process what came before + if hasattr(target, "webfiles"): + return struct() + srcs = [] + deps = [] + if hasattr(ctx.rule.files, "srcs"): + srcs.extend(_ASPECT_SLURP_FILE_TYPE.filter(ctx.rule.files.srcs)) + for attr in ("deps", "sticky_deps", "module_deps"): + value = getattr(ctx.rule.attr, attr, None) + if value: + deps.extend(value) + deps = unfurl(deps, provider="webfiles") webpaths = depset() - manifests = depset(order="topological") - jslibs = depset(order="postorder") + aspect_runfiles = depset(srcs) for dep in deps: webpaths += dep.webfiles.webpaths - manifests += dep.webfiles.manifests - if hasattr(dep.webfiles, "jslibs"): - jslibs += dep.webfiles.jslibs - # process what comes now - srcs = ctx.rule.files.srcs + extra_srcs - jslibs += [src for src in srcs if src.path.endswith(".js")] + if hasattr(dep.webfiles, "aspect_runfiles"): + aspect_runfiles += dep.webfiles.aspect_runfiles manifest_srcs = [] new_webpaths = [] - web_srcs = [] for src in srcs: webpath = "/" + long_path(ctx, src) _add_webpath(ctx, src, webpath, webpaths, new_webpaths, manifest_srcs) - web_srcs.append(src) - # create webfiles manifest - manifest = ctx.new_file(ctx.configuration.bin_dir, - "%s-webfiles.pbtxt" % ctx.label.name) - ctx.file_action( - output=manifest, - content=struct( - label=str(ctx.label), - src=manifest_srcs).to_proto()) - manifests += [manifest] webpaths += new_webpaths + manifest = _make_manifest(ctx, manifest_srcs) + dummy, manifests = _run_webfiles_validator(ctx, srcs, deps, manifest) + aspect_runfiles += [dummy, manifest] return struct( - exports=[] if srcs else deps, webfiles=struct( manifest=manifest, manifests=manifests, webpaths=webpaths, - dummy=manifest, - jslibs=jslibs), - closure_legacy_js_runfiles=(depset(srcs + ctx.rule.files.data) | - extra_transitive | - collect_runfiles(deps) | - collect_runfiles(ctx.rule.files.data))) + dummy=dummy, + aspect_runfiles=aspect_runfiles)) + +def _make_manifest(ctx, src_list): + manifest = _new_file(ctx, "-webfiles.pbtxt") + ctx.file_action( + output=manifest, + content=struct( + label=str(ctx.label), + src=src_list).to_proto()) + return manifest + +def _run_webfiles_validator(ctx, srcs, deps, manifest): + dummy = _new_file(ctx, "-webfiles.ignoreme") + manifests = depset(order="topological") + for dep in deps: + manifests += dep.webfiles.manifests + if srcs: + args = ["WebfilesValidator", + "--dummy", dummy.path, + "--target", manifest.path] + if hasattr(ctx, "attr") and hasattr(ctx.attr, "suppress"): + for category in ctx.attr.suppress: + args.append("--suppress") + args.append(category) + inputs = [manifest] + inputs.extend(srcs) + direct_manifests = depset() + for dep in deps: + inputs.append(dep.webfiles.dummy) + for f in dep.files: + inputs.append(f) + direct_manifests += [dep.webfiles.manifest] + inputs.append(dep.webfiles.manifest) + args.append("--direct_dep") + args.append(dep.webfiles.manifest.path) + for man in difference(manifests, direct_manifests): + inputs.append(man) + args.append("--transitive_dep") + args.append(man.path) + argfile = _new_file(ctx, "-webfiles-checker-args.txt") + ctx.file_action(output=argfile, content="\n".join(args)) + inputs.append(argfile) + ctx.action( + inputs=inputs, + outputs=[dummy], + executable=(getattr(ctx.executable, "_ClosureWorker", None) or + getattr(ctx.executable, "_ClosureWorkerAspect", None)), + arguments=["@@" + argfile.path], + mnemonic="Closure", + execution_requirements={"supports-workers": "1"}, + progress_message="Checking webfiles %s" % ctx.label) + else: + ctx.file_action(output=dummy, content="BOO!") + manifests += [manifest] + return dummy, manifests + +def _new_file(ctx, suffix): + return ctx.new_file(ctx.bin_dir, "%s%s" % (ctx.label.name, suffix)) def _add_webpath(ctx, src, webpath, webpaths, new_webpaths, manifest_srcs): if webpath in new_webpaths: @@ -354,45 +375,45 @@ def _get_strip(ctx): web_aspect = aspect( implementation=_web_aspect_impl, - attr_aspects=["deps"]) + attr_aspects=["deps", "sticky_deps", "module_deps"], + attrs={"_ClosureWorkerAspect": _CLOSURE_WORKER}) ts_web_library = rule( implementation=_ts_web_library, executable=True, - attrs={ + attrs=CLUTZ_ATTRIBUTES + { "path": attr.string(), "srcs": attr.label_list(allow_files=True), - "deps": attr.label_list(aspects=[web_aspect]), + "deps": attr.label_list( + aspects=[ + web_aspect, + clutz_aspect, + legacy_js, + ]), "exports": attr.label_list(), "data": attr.label_list(cfg="data", allow_files=True), "suppress": attr.string_list(), "strip_prefix": attr.string(), "external_assets": attr.string_dict(default={"/_/runfiles": "."}), + "clutz_entry_points": attr.string_list(), "_execrooter": attr.label( - default=Label( - "//tensorflow/tensorboard/scripts:execrooter"), + default=Label("//tensorflow/tensorboard/scripts:execrooter"), executable=True, cfg="host"), "_tsc": attr.label( - default=Label( - "@com_microsoft_typescript//:tsc"), + default=Label("@com_microsoft_typescript//:tsc"), allow_files=True, executable=True, cfg="host"), - "_es6dts": attr.label( - default=Label( - "@com_microsoft_typescript//:lib.es6.d.ts"), + "_default_typings": attr.label( + default=Label("//tensorflow/tensorboard:ts_web_library_default_typings"), allow_files=True), - "_ClosureWorker": attr.label( - default=Label("@io_bazel_rules_closure//java/io/bazel/rules/closure:ClosureWorker"), - executable=True, - cfg="host"), "_WebfilesServer": attr.label( - default=Label( - "@io_bazel_rules_closure//java/io/bazel/rules/closure/webfiles/server:WebfilesServer"), + default=Label("@io_bazel_rules_closure//java/io/bazel/rules/closure/webfiles/server:WebfilesServer"), executable=True, cfg="host"), + "_ClosureWorker": _CLOSURE_WORKER, + "_closure_library_base": CLOSURE_LIBRARY_BASE_ATTR, + "_closure_library_deps": CLOSURE_LIBRARY_DEPS_ATTR, }, - outputs={ - "dummy": "%{name}.ignoreme", - }) + outputs=CLUTZ_OUTPUTS) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 18bc9a8277..1abb3abfc0 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -741,19 +741,27 @@ def tf_workspace(path_prefix="", tf_repo_name=""): ]), ) - ############################################################################## - # TensorBoard JavaScript Production Dependencies + native.new_http_archive( + name = "io_angular_clutz", + build_file = "//third_party:clutz.BUILD", + sha256 = "2981de41d1ff4774b544423da9a2cd8beb3be649e95aef2ef2fd83957300b3fe", + strip_prefix = "clutz-b0db5ade9bb535d387f05292316c422790c9848e", + urls = [ + "http://mirror.bazel.build/github.com/angular/clutz/archive/b0db5ade9bb535d387f05292316c422790c9848e.tar.gz", # 2017-05-22 + "https://github.com/angular/clutz/archive/b0db5ade9bb535d387f05292316c422790c9848e.tar.gz", + ], + ) filegroup_external( name = "com_google_javascript_closure_compiler_externs", licenses = ["notice"], # Apache 2.0 sha256_urls_extract = { - "0ee7b88ed2955b622eaa038bece283e28d0fb5abebfbb80871fc3d0353f0000b": [ - "http://mirror.bazel.build/github.com/google/closure-compiler/archive/v20170423.tar.gz", - "https://github.com/google/closure-compiler/archive/v20170423.tar.gz", + "0f515a6ebfa138490b3c5ea9f3591ea1a7e4a930d3074f18b3eca86084ad9b66": [ + "http://mirror.bazel.build/github.com/google/closure-compiler/archive/b37e6000001b0a6bf4c0be49024ebda14a8711d9.tar.gz", # 2017-06-02 + "https://github.com/google/closure-compiler/archive/b37e6000001b0a6bf4c0be49024ebda14a8711d9.tar.gz", ], }, - strip_prefix = {"v20170423.tar.gz": "closure-compiler-20170423/externs"}, + strip_prefix = {"b37e6000001b0a6bf4c0be49024ebda14a8711d9.tar.gz": "closure-compiler-b37e6000001b0a6bf4c0be49024ebda14a8711d9/externs"}, ) filegroup_external( @@ -761,12 +769,15 @@ def tf_workspace(path_prefix="", tf_repo_name=""): licenses = ["notice"], # Apache 2.0 sha256_urls = { "23baad9a200a717a821c6df504c84d3a893d7ea9102b14876eb80097e3b94292": [ - "http://mirror.bazel.build/raw.githubusercontent.com/google/closure-compiler/0e8dc5597a295ee259e3fecd98d6535dc621232f/contrib/externs/polymer-1.0.js", + "http://mirror.bazel.build/raw.githubusercontent.com/google/closure-compiler/0e8dc5597a295ee259e3fecd98d6535dc621232f/contrib/externs/polymer-1.0.js", # 2017-05-27 "https://raw.githubusercontent.com/google/closure-compiler/0e8dc5597a295ee259e3fecd98d6535dc621232f/contrib/externs/polymer-1.0.js", ], }, ) + ############################################################################## + # TensorBoard JavaScript Production Dependencies + web_library_external( name = "com_lodash", licenses = ["notice"], # MIT diff --git a/third_party/clutz.BUILD b/third_party/clutz.BUILD new file mode 100644 index 0000000000..593b70366a --- /dev/null +++ b/third_party/clutz.BUILD @@ -0,0 +1,44 @@ +# Description: +# Build tool for making TypeScript .d.ts files from Closure JavaScript. + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # MIT + +exports_files([ + "LICENSE", + "src/resources/closure.lib.d.ts", +]) + +JVM_FLAGS = [ + "-Xss20m", # JSCompiler needs big stacks for recursive parsing + "-XX:+UseParallelGC", # Best GC when app isn't latency sensitive +] + +java_binary( + name = "clutz", + srcs = glob(["src/main/java/com/google/javascript/clutz/**/*.java"]), + jvm_flags = JVM_FLAGS, + main_class = "com.google.javascript.clutz.DeclarationGenerator", + deps = [ + "@args4j", + "@com_google_code_findbugs_jsr305", + "@com_google_code_gson", + "@com_google_guava", + "@com_google_javascript_closure_compiler", + ], +) + +java_binary( + name = "gents", + srcs = glob(["src/main/java/com/google/javascript/gents/**/*.java"]), + jvm_flags = JVM_FLAGS, + main_class = "com.google.javascript.gents.TypeScriptGenerator", + deps = [ + "@args4j", + "@com_google_code_findbugs_jsr305", + "@com_google_code_gson", + "@com_google_guava", + "@com_google_javascript_closure_compiler", + ], +) diff --git a/third_party/clutz.bzl b/third_party/clutz.bzl new file mode 100644 index 0000000000..f273c78c79 --- /dev/null +++ b/third_party/clutz.bzl @@ -0,0 +1,77 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build definitions for TypeScript from Closure JavaScript libraries.""" + +load("@io_bazel_rules_closure//closure/private:defs.bzl", + "JS_FILE_TYPE", + "collect_js", + "unfurl") + +CLUTZ_ATTRIBUTES = { + "_clutz": attr.label( + default=Label("@io_angular_clutz//:clutz"), + executable=True, + cfg="host"), + "_clutz_externs": attr.label( + default=Label("@com_google_javascript_closure_compiler_externs"), + allow_files=True), +} + +def extract_dts_from_closure_libraries(ctx): + """Extracts type definitions from closure dependencies. + + This just generates one big .d.ts file for all transitive Closure sources, + and does not pass it down. That means each rule has to duplicate the effort, + but on the other hand allows transitive dependencies on shared rules without + causing duplicate definition errors. + + Args: + ctx: A Skylark context. + Returns: + The generated Clutz typings file, or None if there were no JS deps. + """ + deps = unfurl(ctx.attr.deps, provider="closure_js_library") + js = collect_js(ctx, deps) + if not js.srcs: + return None + js_typings = ctx.new_file(ctx.bin_dir, "%s-js-typings.d.ts" % ctx.label.name) + srcs = depset(JS_FILE_TYPE.filter(ctx.files._clutz_externs)) + js.srcs + args = ["-o", js_typings.path] + for src in srcs: + args.append(src.path) + if getattr(ctx.attr, "clutz_entry_points", None): + args.append("--closure_entry_points") + args.extend(ctx.attr.clutz_entry_points) + ctx.action( + inputs=list(srcs), + outputs=[js_typings], + executable=ctx.executable._clutz, + arguments=args, + mnemonic="Clutz", + progress_message="Running Clutz on %d JS files %s" % ( + len(srcs), ctx.label)) + return js_typings + +################################################################################ +# The following definitions are for API compatibility with internal clutz.bzl + +CLUTZ_OUTPUTS = {} + +def _clutz_aspect_impl(target, ctx): + return struct() + +clutz_aspect = aspect( + implementation=_clutz_aspect_impl, + attr_aspects=["exports"]) |