Include sha hash in python variable __version__
It is necessary to symlink in files from .git/ in order to make
bazel aware of changes to the current head. As it is this is not
completely reliable when git repositories are in a dirty index
state. First class support for bazel git a reported bug but
not a high priority.
./configure sets up the symlinks by calling the gen_git_source.py
a bazel genrule calls gen_git_source.py to generate version_info.cc
Also changed cmake and make to build this properly.
Change: 132328009
diff --git a/configure b/configure
index f4b772c..ce75bb4 100755
--- a/configure
+++ b/configure
@@ -2,6 +2,11 @@
DO_NOT_SUBMIT_WARNING="Unofficial setting. DO NOT SUBMIT!!!"
+# Find out the absolute path to where ./configure resides
+pushd `dirname $0` #> /dev/null
+SOURCE_BASE_DIR=`pwd -P`
+popd > /dev/null
+
## Set up python-related environment settings
while true; do
fromuser=""
@@ -68,6 +73,12 @@
# Invoke python_config and set up symlinks to python includes
(./util/python/python_config.sh --setup "$PYTHON_BIN_PATH";) || exit -1
+# Run the gen_git_source to create links where bazel can track dependencies for
+# git hash propagation
+GEN_GIT_SOURCE=tensorflow/tools/git/gen_git_source.py
+chmod a+x ${GEN_GIT_SOURCE}
+${PYTHON_BIN_PATH} ${GEN_GIT_SOURCE} --configure ${SOURCE_BASE_DIR}
+
## Set up Cuda-related environment settings
while [ "$TF_NEED_CUDA" == "" ]; do
diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index c560948..0995c62 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -168,6 +168,7 @@
"//tensorflow/tools/docker:all_files",
"//tensorflow/tools/docker/notebooks:all_files",
"//tensorflow/tools/docs:all_files",
+ "//tensorflow/tools/git:all_files",
"//tensorflow/tools/proto_text:all_files",
"//tensorflow/tools/test:all_files",
"//tensorflow/user_ops:all_files",
diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake
index 0dbe3c1..cbcd11a 100644
--- a/tensorflow/contrib/cmake/tf_core_framework.cmake
+++ b/tensorflow/contrib/cmake/tf_core_framework.cmake
@@ -180,6 +180,21 @@
boringssl
)
+# Tricky setup to force always rebuilding
+# force_rebuild always runs forcing ${VERSION_INFO_CC} target to run
+# ${VERSION_INFO_CC} would cache, but it depends on a phony never produced
+# target.
+set(VERSION_INFO_CC ${tensorflow_source_dir}/tensorflow/core/util/version_info.cc)
+add_custom_target(force_rebuild_target ALL DEPENDS ${VERSION_INFO_CC})
+add_custom_command(OUTPUT __force_rebuild COMMAND cmake -E echo)
+add_custom_command(OUTPUT
+ ${VERSION_INFO_CC}
+ COMMAND ${tensorflow_source_dir}/tensorflow/tools/git/gen_git_source.py
+ --raw_generate ${VERSION_INFO_CC}
+ DEPENDS __force_rebuild)
+
+set(tf_version_srcs ${tensorflow_source_dir}/tensorflow/core/util/version_info.cc)
+
########################################################
# tf_core_framework library
@@ -211,6 +226,7 @@
add_library(tf_core_framework OBJECT
${tf_core_framework_srcs}
+ ${tf_version_srcs}
${PROTO_TEXT_HDRS}
${PROTO_TEXT_SRCS})
target_include_directories(tf_core_framework PUBLIC
diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile
index 7f2218a..5da8502 100644
--- a/tensorflow/contrib/makefile/Makefile
+++ b/tensorflow/contrib/makefile/Makefile
@@ -421,7 +421,8 @@
$(wildcard tensorflow/core/platform/*/*.cc) \
$(wildcard tensorflow/core/platform/*/*/*.cc) \
$(wildcard tensorflow/core/util/*.cc) \
-$(wildcard tensorflow/core/util/*/*.cc)
+$(wildcard tensorflow/core/util/*/*.cc) \
+tensorflow/core/util/version_info.cc
CORE_CC_EXCLUDE_SRCS := \
$(wildcard tensorflow/core/*/*test.cc) \
$(wildcard tensorflow/core/*/*testutil*) \
@@ -477,6 +478,11 @@
# Rules for target compilation.
+
+.phony_version_info:
+tensorflow/core/util/version_info.cc: .phony_version_info
+ python tensorflow/tools/git/gen_git_source.py --raw_generate $@
+
# Gathers together all the objects we've compiled into a single '.a' archive.
$(LIB_PATH): $(LIB_OBJS)
@mkdir -p $(dir $@)
@@ -553,6 +559,7 @@
# Gets rid of all generated files.
clean:
rm -rf $(MAKEFILE_DIR)/gen
+ rm -rf tensorflow/core/util/version_info.cc
# Gets rid of target files only, leaving the host alone. Also leaves the lib
# directory untouched deliberately, so we can persist multiple architectures
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 5a8b8c8..b100e87 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -68,6 +68,7 @@
)
load("//tensorflow:tensorflow.bzl", "tf_cc_test_gpu")
load("//tensorflow:tensorflow.bzl", "tf_cc_tests_gpu")
+load("//tensorflow:tensorflow.bzl", "tf_version_info_genrule")
# For platform specific build config
load(
@@ -938,6 +939,13 @@
],
)
+tf_version_info_genrule()
+
+cc_library(
+ name = "version_lib",
+ srcs = ["util/version_info.cc"],
+)
+
tf_cuda_library(
name = "framework_internal",
srcs = glob(
@@ -980,6 +988,7 @@
":lib_internal",
":proto_text",
":protos_all_cc",
+ ":version_lib",
"//tensorflow/core/kernels:bounds_check",
"//third_party/eigen3",
],
diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h
index a9b63ea..037e579 100644
--- a/tensorflow/core/public/version.h
+++ b/tensorflow/core/public/version.h
@@ -84,4 +84,12 @@
#define TF_CHECKPOINT_VERSION_MIN_CONSUMER 0
#define TF_CHECKPOINT_VERSION 1
+/// Version query functions (defined in generated version_info.cc)
+
+// Host compiler version (declared elsewhere to be __VERSION__)
+extern const char* tf_compiler_version();
+// The git commit designator when tensorflow was built
+// If no git repository, this will be "internal".
+extern const char* tf_git_version();
+
#endif // TENSORFLOW_CORE_PUBLIC_VERSION_H_
diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index 974d3c2..72a1493 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -245,4 +245,8 @@
'train',
])
-__all__.append('__version__')
+__all__.extend([
+ '__version__',
+ '__git_version__',
+ '__compiler_version__',
+])
diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i
index 87391ff..8afd3f7 100644
--- a/tensorflow/python/client/tf_session.i
+++ b/tensorflow/python/client/tf_session.i
@@ -35,6 +35,12 @@
%constant int GRAPH_DEF_VERSION_MIN_CONSUMER = TF_GRAPH_DEF_VERSION_MIN_CONSUMER;
%constant int GRAPH_DEF_VERSION_MIN_PRODUCER = TF_GRAPH_DEF_VERSION_MIN_PRODUCER;
+// Git version information
+%constant const char* __git_version__ = tf_git_version();
+
+// Compiler
+%constant const char* __compiler_version__ = tf_compiler_version();
+
// Release the Python GIL for the duration of most methods.
%exception {
Py_BEGIN_ALLOW_THREADS;
diff --git a/tensorflow/python/framework/versions.py b/tensorflow/python/framework/versions.py
index 369dd70..63722e0 100644
--- a/tensorflow/python/framework/versions.py
+++ b/tensorflow/python/framework/versions.py
@@ -22,6 +22,9 @@
from tensorflow.python import pywrap_tensorflow
__version__ = pywrap_tensorflow.__version__
+__git_version__ = pywrap_tensorflow.__git_version__
+__compiler_version__ = pywrap_tensorflow.__compiler_version__
+
GRAPH_DEF_VERSION = pywrap_tensorflow.GRAPH_DEF_VERSION
GRAPH_DEF_VERSION_MIN_CONSUMER = (
pywrap_tensorflow.GRAPH_DEF_VERSION_MIN_CONSUMER)
@@ -30,4 +33,5 @@
# Make sure these symbols are exported even though one starts with _.
__all__ = ["__version__", "GRAPH_DEF_VERSION", "GRAPH_DEF_VERSION_MIN_CONSUMER",
- "GRAPH_DEF_VERSION_MIN_PRODUCER"]
+ "GRAPH_DEF_VERSION_MIN_PRODUCER", "__git_version__",
+ "__compiler_version__"]
diff --git a/tensorflow/python/framework/versions_test.py b/tensorflow/python/framework/versions_test.py
index 6042048..025c311 100644
--- a/tensorflow/python/framework/versions_test.py
+++ b/tensorflow/python/framework/versions_test.py
@@ -38,6 +38,9 @@
self.assertLessEqual(0, min_producer)
self.assertLessEqual(min_producer, version)
+ def testGitVersion(self):
+ self.assertEqual(type(tf.__git_version__), str)
+ self.assertEqual(type(tf.__compiler_version__), str)
if __name__ == "__main__":
tf.test.main()
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 3d367d8..565ecdc 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -813,3 +813,18 @@
return ("cat $(SRCS) > $(@) && " +
"echo >> $(@) && " +
"echo " + to_append + " >> $(@)")
+
+
+def tf_version_info_genrule():
+ native.genrule(
+ name = "version_info_gen",
+ srcs = [
+ "//tensorflow/tools/git:gen/spec.json",
+ "//tensorflow/tools/git:gen/head",
+ "//tensorflow/tools/git:gen/branch_ref",
+ ],
+ outs = ["util/version_info.cc"],
+ cmd = "$(location //tensorflow/tools/git:gen_git_source.py) --generate $(SRCS) \"$@\"",
+ local = 1,
+ tools = ["//tensorflow/tools/git:gen_git_source.py"],
+ )
diff --git a/tensorflow/tf_exported_symbols.lds b/tensorflow/tf_exported_symbols.lds
index e7749ab..cb81e89 100644
--- a/tensorflow/tf_exported_symbols.lds
+++ b/tensorflow/tf_exported_symbols.lds
@@ -1,2 +1,3 @@
*tensorflow*
*perftools*gputools*
+*tf_*
diff --git a/tensorflow/tools/git/BUILD b/tensorflow/tools/git/BUILD
new file mode 100644
index 0000000..f502c8d
--- /dev/null
+++ b/tensorflow/tools/git/BUILD
@@ -0,0 +1,28 @@
+# Description:
+# Contains script to generate tensorflow/core/util/version_info.cc
+# Also contains information about git repository deposited by configure
+# in gen/...
+package(default_visibility = ["//tensorflow:internal"])
+
+licenses(["notice"]) # Apache 2.0
+
+exports_files(
+ glob(["gen/*"]) + [
+ "gen_git_source.py",
+ ],
+)
+
+# -----------------------------------------------------------------------------
+# Google-internal targets. These must be at the end for syncrepo.
+
+filegroup(
+ name = "all_files",
+ srcs = glob(
+ ["**/*"],
+ exclude = [
+ "**/METADATA",
+ "**/OWNERS",
+ ],
+ ),
+ visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py
new file mode 100755
index 0000000..6c0770b
--- /dev/null
+++ b/tensorflow/tools/git/gen_git_source.py
@@ -0,0 +1,223 @@
+#!/usr/bin/python
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Help include git hash in tensorflow bazel build.
+
+This creates symlinks from the internal git repository directory so
+that the build system can see changes in the version state. We also
+remember what branch git was on so when the branch changes we can
+detect that the ref file is no longer correct (so we can suggest users
+run ./configure again).
+
+NOTE: this script is only used in opensource.
+
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import argparse
+import json
+import os
+import shutil
+
+
+def parse_branch_ref(filename):
+ """Given a filename of a .git/HEAD file return ref path.
+
+ In particular, if git is in detached head state, this will
+ return None. If git is in attached head, it will return
+ the branch reference. E.g. if on 'master', the HEAD will
+ contain 'ref: refs/heads/master' so 'refs/heads/master'
+ will be returned.
+
+ Example: parse_branch_ref(".git/HEAD")
+ Args:
+ filename: file to treat as a git HEAD file
+ Returns:
+ None if detached head, otherwise ref subpath
+ Raises:
+ RuntimeError: if the HEAD file is unparseable.
+ """
+
+ data = open(filename).read().strip()
+ items = data.split(" ")
+ if len(items) == 1:
+ return None
+ elif len(items) == 2 and items[0] == "ref:":
+ return items[1].strip()
+ else:
+ raise RuntimeError("Git directory has unparseable HEAD")
+
+
+def configure(src_base_path, debug=False):
+ """Configure `src_base_path` to embed git hashes if available."""
+
+ # TODO(aselle): No files generated or symlinked here are deleted by
+ # the build system. I don't know of a way to do it in bazel. It
+ # should only be a problem if somebody moves a sandbox directory
+ # without running ./configure again.
+
+ git_path = os.path.join(src_base_path, ".git")
+ gen_path = os.path.join(src_base_path, "tensorflow", "tools", "git", "gen")
+
+ # Remove and recreate the path
+ if os.path.exists(gen_path):
+ if os.path.isdir(gen_path):
+ shutil.rmtree(gen_path)
+ else:
+ raise RuntimeError("Cannot delete non-directory %s, inspect ",
+ "and remove manually" % gen_path)
+ os.makedirs(gen_path)
+
+ if not os.path.isdir(gen_path):
+ raise RuntimeError("gen_git_source.py: Failed to create dir")
+
+ # file that specifies what the state of the git repo is
+ spec = {}
+
+ # value file names will be mapped to the keys
+ link_map = {"head": None, "branch_ref": None}
+
+ if not os.path.isdir(git_path):
+ # No git directory
+ spec["git"] = False
+ open(os.path.join(gen_path, "head"), "w").write("")
+ open(os.path.join(gen_path, "branch_ref"), "w").write("")
+ else:
+ # Git directory, possibly detached or attached
+ spec["git"] = True
+ spec["path"] = src_base_path
+ git_head_path = os.path.join(git_path, "HEAD")
+ spec["branch"] = parse_branch_ref(git_head_path)
+ link_map["head"] = git_head_path
+ if spec["branch"] is not None:
+ # attached method
+ link_map["branch_ref"] = os.path.join(git_path, *
+ os.path.split(spec["branch"]))
+ # Create symlinks or dummy files
+ for target, src in link_map.items():
+ if src is None:
+ open(os.path.join(gen_path, target), "w").write("")
+ else:
+ os.symlink(src, os.path.join(gen_path, target))
+
+ json.dump(spec, open(os.path.join(gen_path, "spec.json"), "w"), indent=2)
+ if debug:
+ print("gen_git_source.py: list %s" % gen_path)
+ print("gen_git_source.py: %s" + repr(os.listdir(gen_path)))
+ print("gen_git_source.py: spec is %r" % spec)
+
+
+def generate(arglist):
+ """Generate version_info.cc as given `destination_file`.
+
+ Args:
+ arglist: should be a sequence that contains
+ spec, head_symlink, ref_symlink, destination_file.
+
+ `destination_file` is the filename where version_info.cc will be written
+
+ `spec` is a filename where the file contains a JSON dictionary
+ 'git' bool that is true if the source is in a git repo
+ 'path' base path of the source code
+ 'branch' the name of the ref specification of the current branch/tag
+
+ `head_symlink` is a filename to HEAD that is cross-referenced against
+ what is contained in the json branch designation.
+
+ `ref_symlink` is unused in this script but passed, because the build
+ system uses that file to detect when commits happen.
+
+ Raises:
+ RuntimeError: If ./configure needs to be run, RuntimeError will be raised.
+ """
+
+ # unused ref_symlink arg
+ spec, head_symlink, _, dest_file = arglist
+ data = json.load(open(spec))
+ strs = {"tf_compiler_version": "__VERSION__"}
+ if not data["git"]:
+ strs["tf_git_version"] = "internal"
+ else:
+ old_branch = data["branch"]
+ new_branch = parse_branch_ref(head_symlink)
+ if new_branch != old_branch:
+ raise RuntimeError(
+ "Run ./configure again, branch was '%s' but is now '%s'" %
+ (old_branch, new_branch))
+ strs["tf_git_version"] = os.popen(
+ "git -C \"%s\" describe --long --dirty --tags" %
+ (data["path"],)).read().strip()
+ # TODO(aselle): Check for escaping
+ cpp_file = "\n".join("const char* %s() {return \"%s\";}" % (x, y)
+ for x, y in strs.items())
+ open(dest_file, "w").write(cpp_file + "\n")
+
+
+def raw_generate(output_file):
+ """Simple generator used for cmake/make build systems.
+
+ This does not create any symlinks. It requires the build system
+ to build unconditionally.
+
+ Args:
+ output_file: Output filename for the version info cc
+ """
+
+ strs = {"tf_compiler_version": "__VERSION__"}
+ version = os.popen("git describe --long --dirty --tags").read().strip()
+ version = version if version else "unknown"
+ strs["tf_git_version"] = version
+ cpp_file = "\n".join("const char* %s() {return \"%s\";}" % (x, y)
+ for x, y in strs.items())
+ open(output_file, "w").write(cpp_file + "\n")
+
+
+parser = argparse.ArgumentParser(description="""Git hash injection into bazel.
+If used with --configure <path> will search for git directory and put symlinks
+into source so that a bazel genrule can call --generate""")
+
+parser.add_argument(
+ "--debug",
+ type=bool,
+ help="print debugging information about paths",
+ default=False)
+
+parser.add_argument(
+ "--configure", type=str,
+ help="Path to configure as a git repo dependency tracking sentinel")
+
+parser.add_argument(
+ "--generate",
+ type=str,
+ help="Generate given spec-file, HEAD-symlink-file, ref-symlink-file",
+ nargs="+")
+
+parser.add_argument(
+ "--raw_generate",
+ type=str,
+ help="Generate version_info.cc (simpler version used for cmake/make)")
+
+args = parser.parse_args()
+
+if args.configure is not None:
+ configure(args.configure, debug=args.debug)
+elif args.generate is not None:
+ generate(args.generate)
+elif args.raw_generate is not None:
+ raw_generate(args.raw_generate)
+else:
+ raise RuntimeError("--configure or --generate or --raw_generate "
+ "must be used")