diff --git a/WORKSPACE b/WORKSPACE index b4c80d7aadb105e31ffc6cb63c4245c32eed46ce..7ebb1d488c3a6c8eabc32bbd678857b20e6c69a9 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -2,11 +2,11 @@ workspace(name = "org_tensorflow") http_archive( name = "io_bazel_rules_closure", - sha256 = "0e38269c55536196c9b0c82a601c683f114901acb6d55f214e0179a3e188ef2a", - strip_prefix = "rules_closure-1762d8e6964b9f383b47a57888e55489d2432b61", + sha256 = "4be8a887f6f38f883236e77bb25c2da10d506f2bf1a8e5d785c0f35574c74ca4", + strip_prefix = "rules_closure-aac19edc557aec9b603cd7ffe359401264ceff0d", urls = [ - "http://bazel-mirror.storage.googleapis.com/github.com/bazelbuild/rules_closure/archive/1762d8e6964b9f383b47a57888e55489d2432b61.tar.gz", # 2017-05-08 - "https://github.com/bazelbuild/rules_closure/archive/1762d8e6964b9f383b47a57888e55489d2432b61.tar.gz", + "http://bazel-mirror.storage.googleapis.com/github.com/bazelbuild/rules_closure/archive/aac19edc557aec9b603cd7ffe359401264ceff0d.tar.gz", # 2017-05-10 + "https://github.com/bazelbuild/rules_closure/archive/aac19edc557aec9b603cd7ffe359401264ceff0d.tar.gz", ], ) diff --git a/tensorflow/BUILD b/tensorflow/BUILD index c4bfb97e5ec1f06214ad5296956e3007f9ce13fd..981e913f9b2f22d3d2d3ab23241561d6bc817e65 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -366,6 +366,7 @@ filegroup( "//tensorflow/tensorboard/components/vz_sorting:all_files", "//tensorflow/tensorboard/components/vz_sorting/test:all_files", "//tensorflow/tensorboard/components/vz_sorting_d3v4:all_files", + "//tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize:all_files", "//tensorflow/tensorboard/lib:all_files", "//tensorflow/tensorboard/plugins:all_files", "//tensorflow/tensorboard/plugins/projector:all_files", diff --git a/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/BUILD b/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..07fc3a70a704eef0c0a0d6ee24445c5dbb23786b --- /dev/null +++ b/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/BUILD @@ -0,0 +1,23 @@ +package(default_visibility = ["//tensorflow:internal"]) + +licenses(["notice"]) # Apache 2.0 + +java_binary( + name = "Vulcanize", + srcs = ["Vulcanize.java"], + deps = [ + "@com_google_guava", + "@com_google_protobuf_java", + "@io_bazel_rules_closure//closure/compiler", + "@io_bazel_rules_closure//java/io/bazel/rules/closure:webpath", + "@io_bazel_rules_closure//java/io/bazel/rules/closure/webfiles:build_info_java_proto", + "@io_bazel_rules_closure//java/org/jsoup/nodes", + "@org_jsoup", + ], +) + +filegroup( + name = "all_files", + srcs = glob(["**"]), + tags = ["notsan"], +) diff --git a/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java b/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java new file mode 100644 index 0000000000000000000000000000000000000000..e572415856cd7151d04aa2cbd1b8c49678782acd --- /dev/null +++ b/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java @@ -0,0 +1,317 @@ +// Copyright 2017 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.tensorflow.tensorboard.vulcanize; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Verify.verifyNotNull; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.CharMatcher; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.javascript.jscomp.BasicErrorManager; +import com.google.javascript.jscomp.CheckLevel; +import com.google.javascript.jscomp.Compiler; +import com.google.javascript.jscomp.CompilerOptions; +import com.google.javascript.jscomp.CompilerOptions.LanguageMode; +import com.google.javascript.jscomp.CompilerOptions.Reach; +import com.google.javascript.jscomp.JSError; +import com.google.javascript.jscomp.PropertyRenamingPolicy; +import com.google.javascript.jscomp.SourceFile; +import com.google.javascript.jscomp.VariableRenamingPolicy; +import com.google.protobuf.TextFormat; +import io.bazel.rules.closure.Webpath; +import io.bazel.rules.closure.webfiles.BuildInfo.Webfiles; +import io.bazel.rules.closure.webfiles.BuildInfo.WebfilesSource; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Comment; +import org.jsoup.nodes.DataNode; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.nodes.Html5Printer; +import org.jsoup.nodes.Node; +import org.jsoup.nodes.TextNode; +import org.jsoup.parser.Parser; +import org.jsoup.parser.Tag; + +/** Simple one-off solution for TensorBoard vulcanization. */ +public final class Vulcanize { + + private static final Parser parser = Parser.htmlParser(); + private static final Map webfiles = new HashMap<>(); + private static final Set alreadyInlined = new HashSet<>(); + private static final Set legalese = new HashSet<>(); + private static final List licenses = new ArrayList<>(); + private static final List stack = new ArrayList<>(); + private static Webpath outputPath; + private static Node licenseComment; + private static boolean nominify; + + public static void main(String[] args) throws IOException { + Webpath inputPath = Webpath.get(args[0]); + outputPath = Webpath.get(args[1]); + Path output = Paths.get(args[2]); + for (int i = 3; i < args.length; i++) { + Webfiles manifest = loadWebfilesPbtxt(Paths.get(args[i])); + for (WebfilesSource src : manifest.getSrcList()) { + webfiles.put(Webpath.get(src.getWebpath()), Paths.get(src.getPath())); + } + } + stack.add(inputPath); + Document document = parse(Files.readAllBytes(webfiles.get(inputPath))); + transform(document); + if (licenseComment != null) { + licenseComment.attr("comment", String.format("\n%s\n", Joiner.on("\n\n").join(licenses))); + } + Files.write( + output, + Html5Printer.stringify(document).getBytes(UTF_8), + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + } + + private static void transform(Node root) throws IOException { + Node node = checkNotNull(root); + Node newNode; + while (true) { + newNode = enterNode(node); + if (node.equals(root)) { + root = newNode; + } + node = newNode; + if (node.childNodeSize() > 0) { + node = node.childNode(0); + } else { + while (true) { + newNode = leaveNode(node); + if (node.equals(root)) { + root = newNode; + } + node = newNode; + if (node.equals(root)) { + return; + } + Node next = node.nextSibling(); + if (next == null) { + if (node.parentNode() == null) { + return; + } + node = verifyNotNull(node.parentNode(), "unexpected root: %s", node); + } else { + node = next; + break; + } + } + } + } + } + + private static Node enterNode(Node node) throws IOException { + Node newNode = node; + if (node instanceof Element) { + if (node.nodeName().equals("link") && node.attr("rel").equals("import")) { + // Inline HTML. + Webpath href = me().lookup(Webpath.get(node.attr("href"))); + if (alreadyInlined.add(href)) { + newNode = + parse(Files.readAllBytes(checkNotNull(webfiles.get(href), "%s in %s", href, me()))); + stack.add(href); + node.replaceWith(newNode); + } else { + newNode = new TextNode("", node.baseUri()); + node.replaceWith(newNode); + } + } else if (node.nodeName().equals("script")) { + nominify = node.hasAttr("nominify"); + node.removeAttr("nominify"); + Webpath src; + String script; + if (node.attr("src").isEmpty()) { + // Minify JavaScript. + StringBuilder sb = new StringBuilder(); + for (Node child : node.childNodes()) { + if (child instanceof DataNode) { + sb.append(((DataNode) child).getWholeData()); + } + } + src = me(); + script = sb.toString(); + } else { + // Inline JavaScript. + src = me().lookup(Webpath.get(node.attr("src"))); + Path other = webfiles.get(src); + if (other != null) { + script = new String(Files.readAllBytes(other), UTF_8); + node.removeAttr("src"); + } else { + src = me(); + script = ""; + } + } + script = minify(src, script); + newNode = + new Element(Tag.valueOf("script"), node.baseUri(), node.attributes()) + .appendChild(new DataNode(script, node.baseUri())); + node.replaceWith(newNode); + } else if (node.nodeName().equals("link") + && node.attr("rel").equals("stylesheet") + && !node.attr("href").isEmpty()) { + // Inline CSS. + Webpath href = me().lookup(Webpath.get(node.attr("href"))); + Path other = webfiles.get(href); + if (other != null) { + newNode = + new Element(Tag.valueOf("style"), node.baseUri(), node.attributes()) + .appendChild( + new DataNode(new String(Files.readAllBytes(other), UTF_8), node.baseUri())); + newNode.removeAttr("rel"); + newNode.removeAttr("href"); + node.replaceWith(newNode); + } + } + rootifyAttribute(newNode, "href"); + rootifyAttribute(newNode, "src"); + rootifyAttribute(newNode, "action"); + rootifyAttribute(newNode, "assetpath"); + } else if (node instanceof Comment) { + String text = ((Comment) node).getData(); + if (text.contains("@license")) { + handleLicense(text); + if (licenseComment == null) { + licenseComment = node; + } else { + newNode = new TextNode("", node.baseUri()); + node.replaceWith(newNode); + } + } else { + newNode = new TextNode("", node.baseUri()); + node.replaceWith(newNode); + } + } + return newNode; + } + + private static String minify(Webpath src, String script) { + if (nominify) { + return script; + } + Compiler compiler = new Compiler(new JsPrintlessErrorManager()); + CompilerOptions options = new CompilerOptions(); + options.skipAllCompilerPasses(); // too lazy to get externs + options.setLanguageIn(LanguageMode.ECMASCRIPT_2016); + options.setLanguageOut(LanguageMode.ECMASCRIPT5); + options.setContinueAfterErrors(true); + options.setManageClosureDependencies(false); + options.setRenamingPolicy(VariableRenamingPolicy.LOCAL, PropertyRenamingPolicy.OFF); + options.setShadowVariables(true); + options.setInlineVariables(Reach.LOCAL_ONLY); + options.setFlowSensitiveInlineVariables(true); + options.setInlineFunctions(Reach.LOCAL_ONLY); + options.setAssumeClosuresOnlyCaptureReferences(false); + options.setCheckGlobalThisLevel(CheckLevel.OFF); + options.setFoldConstants(true); + options.setCoalesceVariableNames(true); + options.setDeadAssignmentElimination(true); + options.setCollapseVariableDeclarations(true); + options.setConvertToDottedProperties(true); + options.setLabelRenaming(true); + options.setRemoveDeadCode(true); + options.setOptimizeArgumentsArray(true); + options.setRemoveUnusedVariables(Reach.LOCAL_ONLY); + options.setCollapseObjectLiterals(true); + options.setProtectHiddenSideEffects(true); + //options.setPrettyPrint(true); + compiler.disableThreads(); + compiler.compile( + ImmutableList.of(), + ImmutableList.of(SourceFile.fromCode(src.toString(), script)), + options); + return compiler.toSource(); + } + + private static void handleLicense(String text) { + if (legalese.add(CharMatcher.whitespace().removeFrom(text))) { + licenses.add(CharMatcher.anyOf("\r\n").trimFrom(text)); + } + } + + private static Node leaveNode(Node node) { + if (node instanceof Document) { + stack.remove(stack.size() - 1); + } + return node; + } + + private static Webpath me() { + return Iterables.getLast(stack); + } + + private static void rootifyAttribute(Node node, String attribute) { + String value = node.attr(attribute); + if (value.isEmpty()) { + return; + } + Webpath uri = Webpath.get(value); + if (webfiles.containsKey(uri)) { + node.attr(attribute, outputPath.getParent().relativize(uri).toString()); + } + } + + private static Document parse(byte[] bytes) { + return parse(new ByteArrayInputStream(bytes)); + } + + private static Document parse(InputStream input) { + Document document; + try { + document = Jsoup.parse(input, null, "", parser); + } catch (IOException e) { + throw new AssertionError("I/O error when parsing byte array D:", e); + } + document.outputSettings().indentAmount(0); + document.outputSettings().prettyPrint(false); + return document; + } + + private static Webfiles loadWebfilesPbtxt(Path path) throws IOException { + Webfiles.Builder build = Webfiles.newBuilder(); + TextFormat.getParser().merge(new String(Files.readAllBytes(path), UTF_8), build); + return build.build(); + } + + private static final class JsPrintlessErrorManager extends BasicErrorManager { + + @Override + public void println(CheckLevel level, JSError error) {} + + @Override + public void printSummary() {} + } +} diff --git a/tensorflow/tensorboard/vulcanize.bzl b/tensorflow/tensorboard/vulcanize.bzl new file mode 100644 index 0000000000000000000000000000000000000000..f7d88047afca8556642fa24bed710c79a1285fd3 --- /dev/null +++ b/tensorflow/tensorboard/vulcanize.bzl @@ -0,0 +1,100 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@io_bazel_rules_closure//closure/private:defs.bzl", "unfurl", "long_path") + +def _tensorboard_html_binary(ctx): + deps = unfurl(ctx.attr.deps, provider="webfiles") + manifests = set(order="link") + files = set() + for dep in deps: + manifests += dep.webfiles.manifests + files += dep.data_runfiles.files + + # vulcanize + ctx.action( + inputs=list(manifests + files), + outputs=[ctx.outputs.html], + executable=ctx.executable._Vulcanize, + arguments=([ctx.attr.input_path, + ctx.attr.output_path, + ctx.outputs.html.path] + + [m.path for m in manifests]), + progress_message="Vulcanizing %s" % ctx.attr.input_path) + + # webfiles manifest + manifest_srcs = [struct(path=ctx.outputs.html.path, + longpath=long_path(ctx, ctx.outputs.html), + webpath=ctx.attr.output_path)] + manifest = ctx.new_file(ctx.configuration.bin_dir, + "%s.pbtxt" % ctx.label.name) + ctx.file_action( + output=manifest, + content=struct( + label=str(ctx.label), + src=manifest_srcs).to_proto()) + manifests += [manifest] + + # webfiles server + params = struct( + label=str(ctx.label), + bind="[::]:6006", + manifest=[long_path(ctx, man) for man in manifests], + external_asset=[struct(webpath=k, path=v) + for k, v in ctx.attr.external_assets.items()]) + params_file = ctx.new_file(ctx.configuration.bin_dir, + "%s_server_params.pbtxt" % ctx.label.name) + ctx.file_action(output=params_file, content=params.to_proto()) + ctx.file_action( + executable=True, + output=ctx.outputs.executable, + content="#!/bin/sh\nexec %s %s" % ( + ctx.executable._WebfilesServer.short_path, + long_path(ctx, params_file))) + + transitive_runfiles = set() + transitive_runfiles += ctx.attr._WebfilesServer.data_runfiles.files + for dep in deps: + transitive_runfiles += dep.data_runfiles.files + return struct( + files=set([ctx.outputs.html]), + runfiles=ctx.runfiles( + files=ctx.files.data + [manifest, + params_file, + ctx.outputs.html, + ctx.outputs.executable], + transitive_files=transitive_runfiles)) + +tensorboard_html_binary = rule( + implementation=_tensorboard_html_binary, + executable=True, + attrs={ + "input_path": attr.string(mandatory=True), + "output_path": attr.string(mandatory=True), + "data": attr.label_list(cfg="data", allow_files=True), + "deps": attr.label_list(providers=["webfiles"], mandatory=True), + "external_assets": attr.string_dict(default={"/_/runfiles": "."}), + "_Vulcanize": attr.label( + default=Label("//tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize:Vulcanize"), + executable=True, + cfg="host"), + "_WebfilesServer": attr.label( + default=Label( + "@io_bazel_rules_closure//java/io/bazel/rules/closure/webfiles/server:WebfilesServer"), + executable=True, + cfg="host"), + }, + outputs={ + "html": "%{name}.html", + })