Introduce one-off solution for vulcanization
This is a simple Java script that inlines HTML, CSS, and JavaScript
with minification. It tries its best to preserve @license data.
The Skylark rule behaves the same as web_library(). It can be bazel
run to get the development web server. It outputs the protobuf, because
protobuf is awesome.
PiperOrigin-RevId: 155686060
diff --git a/WORKSPACE b/WORKSPACE
index b4c80d7..7ebb1d4 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -2,11 +2,11 @@
http_archive(
name = "io_bazel_rules_closure",
- sha256 = "0e38269c55536196c9b0c82a601c683f114901acb6d55f214e0179a3e188ef2a",
- strip_prefix = "rules_closure-1762d8e6964b9f383b47a57888e55489d2432b61",
+ sha256 = "4be8a887f6f38f883236e77bb25c2da10d506f2bf1a8e5d785c0f35574c74ca4",
+ strip_prefix = "rules_closure-aac19edc557aec9b603cd7ffe359401264ceff0d",
urls = [
- "http://bazel-mirror.storage.googleapis.com/github.com/bazelbuild/rules_closure/archive/1762d8e6964b9f383b47a57888e55489d2432b61.tar.gz", # 2017-05-08
- "https://github.com/bazelbuild/rules_closure/archive/1762d8e6964b9f383b47a57888e55489d2432b61.tar.gz",
+ "http://bazel-mirror.storage.googleapis.com/github.com/bazelbuild/rules_closure/archive/aac19edc557aec9b603cd7ffe359401264ceff0d.tar.gz", # 2017-05-10
+ "https://github.com/bazelbuild/rules_closure/archive/aac19edc557aec9b603cd7ffe359401264ceff0d.tar.gz",
],
)
diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index c4bfb97..981e913 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -366,6 +366,7 @@
"//tensorflow/tensorboard/components/vz_sorting:all_files",
"//tensorflow/tensorboard/components/vz_sorting/test:all_files",
"//tensorflow/tensorboard/components/vz_sorting_d3v4:all_files",
+ "//tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize:all_files",
"//tensorflow/tensorboard/lib:all_files",
"//tensorflow/tensorboard/plugins:all_files",
"//tensorflow/tensorboard/plugins/projector:all_files",
diff --git a/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/BUILD b/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/BUILD
new file mode 100644
index 0000000..07fc3a7
--- /dev/null
+++ b/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/BUILD
@@ -0,0 +1,23 @@
+package(default_visibility = ["//tensorflow:internal"])
+
+licenses(["notice"]) # Apache 2.0
+
+java_binary(
+ name = "Vulcanize",
+ srcs = ["Vulcanize.java"],
+ deps = [
+ "@com_google_guava",
+ "@com_google_protobuf_java",
+ "@io_bazel_rules_closure//closure/compiler",
+ "@io_bazel_rules_closure//java/io/bazel/rules/closure:webpath",
+ "@io_bazel_rules_closure//java/io/bazel/rules/closure/webfiles:build_info_java_proto",
+ "@io_bazel_rules_closure//java/org/jsoup/nodes",
+ "@org_jsoup",
+ ],
+)
+
+filegroup(
+ name = "all_files",
+ srcs = glob(["**"]),
+ tags = ["notsan"],
+)
diff --git a/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java b/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java
new file mode 100644
index 0000000..e572415
--- /dev/null
+++ b/tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java
@@ -0,0 +1,317 @@
+// Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.tensorflow.tensorboard.vulcanize;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Verify.verifyNotNull;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.base.CharMatcher;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.javascript.jscomp.BasicErrorManager;
+import com.google.javascript.jscomp.CheckLevel;
+import com.google.javascript.jscomp.Compiler;
+import com.google.javascript.jscomp.CompilerOptions;
+import com.google.javascript.jscomp.CompilerOptions.LanguageMode;
+import com.google.javascript.jscomp.CompilerOptions.Reach;
+import com.google.javascript.jscomp.JSError;
+import com.google.javascript.jscomp.PropertyRenamingPolicy;
+import com.google.javascript.jscomp.SourceFile;
+import com.google.javascript.jscomp.VariableRenamingPolicy;
+import com.google.protobuf.TextFormat;
+import io.bazel.rules.closure.Webpath;
+import io.bazel.rules.closure.webfiles.BuildInfo.Webfiles;
+import io.bazel.rules.closure.webfiles.BuildInfo.WebfilesSource;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Comment;
+import org.jsoup.nodes.DataNode;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.nodes.Html5Printer;
+import org.jsoup.nodes.Node;
+import org.jsoup.nodes.TextNode;
+import org.jsoup.parser.Parser;
+import org.jsoup.parser.Tag;
+
+/** Simple one-off solution for TensorBoard vulcanization. */
+public final class Vulcanize {
+
+ private static final Parser parser = Parser.htmlParser();
+ private static final Map<Webpath, Path> webfiles = new HashMap<>();
+ private static final Set<Webpath> alreadyInlined = new HashSet<>();
+ private static final Set<String> legalese = new HashSet<>();
+ private static final List<String> licenses = new ArrayList<>();
+ private static final List<Webpath> stack = new ArrayList<>();
+ private static Webpath outputPath;
+ private static Node licenseComment;
+ private static boolean nominify;
+
+ public static void main(String[] args) throws IOException {
+ Webpath inputPath = Webpath.get(args[0]);
+ outputPath = Webpath.get(args[1]);
+ Path output = Paths.get(args[2]);
+ for (int i = 3; i < args.length; i++) {
+ Webfiles manifest = loadWebfilesPbtxt(Paths.get(args[i]));
+ for (WebfilesSource src : manifest.getSrcList()) {
+ webfiles.put(Webpath.get(src.getWebpath()), Paths.get(src.getPath()));
+ }
+ }
+ stack.add(inputPath);
+ Document document = parse(Files.readAllBytes(webfiles.get(inputPath)));
+ transform(document);
+ if (licenseComment != null) {
+ licenseComment.attr("comment", String.format("\n%s\n", Joiner.on("\n\n").join(licenses)));
+ }
+ Files.write(
+ output,
+ Html5Printer.stringify(document).getBytes(UTF_8),
+ StandardOpenOption.WRITE,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING);
+ }
+
+ private static void transform(Node root) throws IOException {
+ Node node = checkNotNull(root);
+ Node newNode;
+ while (true) {
+ newNode = enterNode(node);
+ if (node.equals(root)) {
+ root = newNode;
+ }
+ node = newNode;
+ if (node.childNodeSize() > 0) {
+ node = node.childNode(0);
+ } else {
+ while (true) {
+ newNode = leaveNode(node);
+ if (node.equals(root)) {
+ root = newNode;
+ }
+ node = newNode;
+ if (node.equals(root)) {
+ return;
+ }
+ Node next = node.nextSibling();
+ if (next == null) {
+ if (node.parentNode() == null) {
+ return;
+ }
+ node = verifyNotNull(node.parentNode(), "unexpected root: %s", node);
+ } else {
+ node = next;
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ private static Node enterNode(Node node) throws IOException {
+ Node newNode = node;
+ if (node instanceof Element) {
+ if (node.nodeName().equals("link") && node.attr("rel").equals("import")) {
+ // Inline HTML.
+ Webpath href = me().lookup(Webpath.get(node.attr("href")));
+ if (alreadyInlined.add(href)) {
+ newNode =
+ parse(Files.readAllBytes(checkNotNull(webfiles.get(href), "%s in %s", href, me())));
+ stack.add(href);
+ node.replaceWith(newNode);
+ } else {
+ newNode = new TextNode("", node.baseUri());
+ node.replaceWith(newNode);
+ }
+ } else if (node.nodeName().equals("script")) {
+ nominify = node.hasAttr("nominify");
+ node.removeAttr("nominify");
+ Webpath src;
+ String script;
+ if (node.attr("src").isEmpty()) {
+ // Minify JavaScript.
+ StringBuilder sb = new StringBuilder();
+ for (Node child : node.childNodes()) {
+ if (child instanceof DataNode) {
+ sb.append(((DataNode) child).getWholeData());
+ }
+ }
+ src = me();
+ script = sb.toString();
+ } else {
+ // Inline JavaScript.
+ src = me().lookup(Webpath.get(node.attr("src")));
+ Path other = webfiles.get(src);
+ if (other != null) {
+ script = new String(Files.readAllBytes(other), UTF_8);
+ node.removeAttr("src");
+ } else {
+ src = me();
+ script = "";
+ }
+ }
+ script = minify(src, script);
+ newNode =
+ new Element(Tag.valueOf("script"), node.baseUri(), node.attributes())
+ .appendChild(new DataNode(script, node.baseUri()));
+ node.replaceWith(newNode);
+ } else if (node.nodeName().equals("link")
+ && node.attr("rel").equals("stylesheet")
+ && !node.attr("href").isEmpty()) {
+ // Inline CSS.
+ Webpath href = me().lookup(Webpath.get(node.attr("href")));
+ Path other = webfiles.get(href);
+ if (other != null) {
+ newNode =
+ new Element(Tag.valueOf("style"), node.baseUri(), node.attributes())
+ .appendChild(
+ new DataNode(new String(Files.readAllBytes(other), UTF_8), node.baseUri()));
+ newNode.removeAttr("rel");
+ newNode.removeAttr("href");
+ node.replaceWith(newNode);
+ }
+ }
+ rootifyAttribute(newNode, "href");
+ rootifyAttribute(newNode, "src");
+ rootifyAttribute(newNode, "action");
+ rootifyAttribute(newNode, "assetpath");
+ } else if (node instanceof Comment) {
+ String text = ((Comment) node).getData();
+ if (text.contains("@license")) {
+ handleLicense(text);
+ if (licenseComment == null) {
+ licenseComment = node;
+ } else {
+ newNode = new TextNode("", node.baseUri());
+ node.replaceWith(newNode);
+ }
+ } else {
+ newNode = new TextNode("", node.baseUri());
+ node.replaceWith(newNode);
+ }
+ }
+ return newNode;
+ }
+
+ private static String minify(Webpath src, String script) {
+ if (nominify) {
+ return script;
+ }
+ Compiler compiler = new Compiler(new JsPrintlessErrorManager());
+ CompilerOptions options = new CompilerOptions();
+ options.skipAllCompilerPasses(); // too lazy to get externs
+ options.setLanguageIn(LanguageMode.ECMASCRIPT_2016);
+ options.setLanguageOut(LanguageMode.ECMASCRIPT5);
+ options.setContinueAfterErrors(true);
+ options.setManageClosureDependencies(false);
+ options.setRenamingPolicy(VariableRenamingPolicy.LOCAL, PropertyRenamingPolicy.OFF);
+ options.setShadowVariables(true);
+ options.setInlineVariables(Reach.LOCAL_ONLY);
+ options.setFlowSensitiveInlineVariables(true);
+ options.setInlineFunctions(Reach.LOCAL_ONLY);
+ options.setAssumeClosuresOnlyCaptureReferences(false);
+ options.setCheckGlobalThisLevel(CheckLevel.OFF);
+ options.setFoldConstants(true);
+ options.setCoalesceVariableNames(true);
+ options.setDeadAssignmentElimination(true);
+ options.setCollapseVariableDeclarations(true);
+ options.setConvertToDottedProperties(true);
+ options.setLabelRenaming(true);
+ options.setRemoveDeadCode(true);
+ options.setOptimizeArgumentsArray(true);
+ options.setRemoveUnusedVariables(Reach.LOCAL_ONLY);
+ options.setCollapseObjectLiterals(true);
+ options.setProtectHiddenSideEffects(true);
+ //options.setPrettyPrint(true);
+ compiler.disableThreads();
+ compiler.compile(
+ ImmutableList.<SourceFile>of(),
+ ImmutableList.of(SourceFile.fromCode(src.toString(), script)),
+ options);
+ return compiler.toSource();
+ }
+
+ private static void handleLicense(String text) {
+ if (legalese.add(CharMatcher.whitespace().removeFrom(text))) {
+ licenses.add(CharMatcher.anyOf("\r\n").trimFrom(text));
+ }
+ }
+
+ private static Node leaveNode(Node node) {
+ if (node instanceof Document) {
+ stack.remove(stack.size() - 1);
+ }
+ return node;
+ }
+
+ private static Webpath me() {
+ return Iterables.getLast(stack);
+ }
+
+ private static void rootifyAttribute(Node node, String attribute) {
+ String value = node.attr(attribute);
+ if (value.isEmpty()) {
+ return;
+ }
+ Webpath uri = Webpath.get(value);
+ if (webfiles.containsKey(uri)) {
+ node.attr(attribute, outputPath.getParent().relativize(uri).toString());
+ }
+ }
+
+ private static Document parse(byte[] bytes) {
+ return parse(new ByteArrayInputStream(bytes));
+ }
+
+ private static Document parse(InputStream input) {
+ Document document;
+ try {
+ document = Jsoup.parse(input, null, "", parser);
+ } catch (IOException e) {
+ throw new AssertionError("I/O error when parsing byte array D:", e);
+ }
+ document.outputSettings().indentAmount(0);
+ document.outputSettings().prettyPrint(false);
+ return document;
+ }
+
+ private static Webfiles loadWebfilesPbtxt(Path path) throws IOException {
+ Webfiles.Builder build = Webfiles.newBuilder();
+ TextFormat.getParser().merge(new String(Files.readAllBytes(path), UTF_8), build);
+ return build.build();
+ }
+
+ private static final class JsPrintlessErrorManager extends BasicErrorManager {
+
+ @Override
+ public void println(CheckLevel level, JSError error) {}
+
+ @Override
+ public void printSummary() {}
+ }
+}
diff --git a/tensorflow/tensorboard/vulcanize.bzl b/tensorflow/tensorboard/vulcanize.bzl
new file mode 100644
index 0000000..f7d8804
--- /dev/null
+++ b/tensorflow/tensorboard/vulcanize.bzl
@@ -0,0 +1,100 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@io_bazel_rules_closure//closure/private:defs.bzl", "unfurl", "long_path")
+
+def _tensorboard_html_binary(ctx):
+ deps = unfurl(ctx.attr.deps, provider="webfiles")
+ manifests = set(order="link")
+ files = set()
+ for dep in deps:
+ manifests += dep.webfiles.manifests
+ files += dep.data_runfiles.files
+
+ # vulcanize
+ ctx.action(
+ inputs=list(manifests + files),
+ outputs=[ctx.outputs.html],
+ executable=ctx.executable._Vulcanize,
+ arguments=([ctx.attr.input_path,
+ ctx.attr.output_path,
+ ctx.outputs.html.path] +
+ [m.path for m in manifests]),
+ progress_message="Vulcanizing %s" % ctx.attr.input_path)
+
+ # webfiles manifest
+ manifest_srcs = [struct(path=ctx.outputs.html.path,
+ longpath=long_path(ctx, ctx.outputs.html),
+ webpath=ctx.attr.output_path)]
+ manifest = ctx.new_file(ctx.configuration.bin_dir,
+ "%s.pbtxt" % ctx.label.name)
+ ctx.file_action(
+ output=manifest,
+ content=struct(
+ label=str(ctx.label),
+ src=manifest_srcs).to_proto())
+ manifests += [manifest]
+
+ # webfiles server
+ params = struct(
+ label=str(ctx.label),
+ bind="[::]:6006",
+ manifest=[long_path(ctx, man) for man in manifests],
+ external_asset=[struct(webpath=k, path=v)
+ for k, v in ctx.attr.external_assets.items()])
+ params_file = ctx.new_file(ctx.configuration.bin_dir,
+ "%s_server_params.pbtxt" % ctx.label.name)
+ ctx.file_action(output=params_file, content=params.to_proto())
+ ctx.file_action(
+ executable=True,
+ output=ctx.outputs.executable,
+ content="#!/bin/sh\nexec %s %s" % (
+ ctx.executable._WebfilesServer.short_path,
+ long_path(ctx, params_file)))
+
+ transitive_runfiles = set()
+ transitive_runfiles += ctx.attr._WebfilesServer.data_runfiles.files
+ for dep in deps:
+ transitive_runfiles += dep.data_runfiles.files
+ return struct(
+ files=set([ctx.outputs.html]),
+ runfiles=ctx.runfiles(
+ files=ctx.files.data + [manifest,
+ params_file,
+ ctx.outputs.html,
+ ctx.outputs.executable],
+ transitive_files=transitive_runfiles))
+
+tensorboard_html_binary = rule(
+ implementation=_tensorboard_html_binary,
+ executable=True,
+ attrs={
+ "input_path": attr.string(mandatory=True),
+ "output_path": attr.string(mandatory=True),
+ "data": attr.label_list(cfg="data", allow_files=True),
+ "deps": attr.label_list(providers=["webfiles"], mandatory=True),
+ "external_assets": attr.string_dict(default={"/_/runfiles": "."}),
+ "_Vulcanize": attr.label(
+ default=Label("//tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize:Vulcanize"),
+ executable=True,
+ cfg="host"),
+ "_WebfilesServer": attr.label(
+ default=Label(
+ "@io_bazel_rules_closure//java/io/bazel/rules/closure/webfiles/server:WebfilesServer"),
+ executable=True,
+ cfg="host"),
+ },
+ outputs={
+ "html": "%{name}.html",
+ })