Allow to download clang and use clang for CPU builds.
Previously we only allowed to download clang when doing GPU builds.
The added skylark files use bazel's autoconf scripts, which were only added in
0.10.0. To provide nice error message for older versions of bazel
(i.e. 'version is less than 0.10' vs 'can't load @bazel_tools/cpp/...'), we
move the bazel version check into WORKSPACE file from workspace.bzl.
PiperOrigin-RevId: 190050798
diff --git a/WORKSPACE b/WORKSPACE
index 1e38a9a..11c5cdb 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -14,6 +14,12 @@
closure_repositories()
+# We must check the bazel version before trying to parse any other BUILD
+# files, in case the parsing of those build files depends on the bazel
+# version we require here.
+load("//tensorflow:version_check.bzl", "check_bazel_version_at_least")
+check_bazel_version_at_least("0.10.0")
+
load("//tensorflow:workspace.bzl", "tf_workspace")
# Uncomment and update the paths in these entries to build the Android demo.
diff --git a/configure.py b/configure.py
index 7d61c2e..ea732c6 100644
--- a/configure.py
+++ b/configure.py
@@ -524,7 +524,7 @@
def set_tf_download_clang(environ_cp):
"""Set TF_DOWNLOAD_CLANG action_env."""
- question = 'Do you want to download a fresh release of clang? (Experimental)'
+ question = 'Do you wish to download a fresh release of clang? (Experimental)'
yes_reply = 'Clang will be downloaded and used to compile tensorflow.'
no_reply = 'Clang will not be downloaded.'
set_action_env_var(
@@ -1380,7 +1380,7 @@
# environment variables.
environ_cp = dict(os.environ)
- check_bazel_version('0.5.4')
+ check_bazel_version('0.10.0')
reset_tf_configure_bazelrc(args.workspace)
cleanup_makefile()
@@ -1397,6 +1397,9 @@
environ_cp['TF_NEED_OPENCL'] = '0'
environ_cp['TF_CUDA_CLANG'] = '0'
environ_cp['TF_NEED_TENSORRT'] = '0'
+ # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on
+ # Windows.
+ environ_cp['TF_DOWNLOAD_CLANG'] = '0'
if is_macos():
environ_cp['TF_NEED_JEMALLOC'] = '0'
@@ -1444,16 +1447,8 @@
set_tf_cuda_clang(environ_cp)
if environ_cp.get('TF_CUDA_CLANG') == '1':
- if not is_windows():
- # Ask if we want to download clang release while building.
- set_tf_download_clang(environ_cp)
- else:
- # We use bazel's generated crosstool on Windows and there is no
- # way to provide downloaded toolchain for that yet.
- # TODO(ibiryukov): Investigate using clang as a cuda compiler on
- # Windows.
- environ_cp['TF_DOWNLOAD_CLANG'] = '0'
-
+ # Ask whether we should download the clang toolchain.
+ set_tf_download_clang(environ_cp)
if environ_cp.get('TF_DOWNLOAD_CLANG') != '1':
# Set up which clang we should use as the cuda / host compiler.
set_clang_cuda_compiler_path(environ_cp)
@@ -1463,6 +1458,13 @@
if not is_windows():
set_gcc_host_compiler_path(environ_cp)
set_other_cuda_vars(environ_cp)
+ else:
+ # CUDA not required. Ask whether we should download the clang toolchain and
+ # use it for the CPU build.
+ set_tf_download_clang(environ_cp)
+ if environ_cp.get('TF_DOWNLOAD_CLANG') == '1':
+ write_to_bazelrc('build --config=download_clang')
+ write_to_bazelrc('test --config=download_clang')
set_build_var(environ_cp, 'TF_NEED_MPI', 'MPI', 'with_mpi_support', False)
if environ_cp.get('TF_NEED_MPI') == '1':
diff --git a/tensorflow/version_check.bzl b/tensorflow/version_check.bzl
new file mode 100644
index 0000000..79e721d
--- /dev/null
+++ b/tensorflow/version_check.bzl
@@ -0,0 +1,48 @@
+""" Helpers to check minimum version of bazel."""
+
+def _extract_version_number(bazel_version):
+ """Extracts the semantic version number from a version string
+
+ Args:
+ bazel_version: the version string that begins with the semantic version
+ e.g. "1.2.3rc1 abc1234" where "abc1234" is a commit hash.
+
+ Returns:
+ The semantic version string, like "1.2.3".
+ """
+ for i in range(len(bazel_version)):
+ c = bazel_version[i]
+ if not (c.isdigit() or c == "."):
+ return bazel_version[:i]
+ return bazel_version
+
+# Parse the bazel version string from `native.bazel_version`.
+# e.g.
+# "0.10.0rc1 abc123d" => (0, 10, 0)
+# "0.3.0" => (0, 3, 0)
+def _parse_bazel_version(bazel_version):
+ """Parses a version string into a 3-tuple of ints
+
+ int tuples can be compared directly using binary operators (<, >).
+
+ Args:
+ bazel_version: the Bazel version string
+
+ Returns:
+ An int 3-tuple of a (major, minor, patch) version.
+ """
+
+ version = _extract_version_number(bazel_version)
+ return tuple([int(n) for n in version.split(".")])
+
+def check_bazel_version_at_least(minimum_bazel_version):
+ if "bazel_version" not in dir(native):
+ fail("\nCurrent Bazel version is lower than 0.2.1, expected at least %s\n" % minimum_bazel_version)
+ elif not native.bazel_version:
+ print("\nCurrent Bazel is not a release version, cannot check for compatibility.")
+ print("Make sure that you are running at least Bazel %s.\n" % minimum_bazel_version)
+ return
+
+ if _parse_bazel_version(native.bazel_version) < _parse_bazel_version(minimum_bazel_version):
+ fail("\nCurrent Bazel version is {}, expected at least {}\n".format(
+ native.bazel_version, minimum_bazel_version))
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 675acbe..ebb9e94 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -10,65 +10,18 @@
load("//third_party/toolchains/clang6:repo.bzl", "clang6_configure")
load("//third_party/toolchains/cpus/arm:arm_compiler_configure.bzl", "arm_compiler_configure")
load("//third_party:repo.bzl", "tf_http_archive")
+load("//third_party/clang_toolchain:cc_configure_clang.bzl", "cc_download_clang_toolchain")
load("@io_bazel_rules_closure//closure/private:java_import_external.bzl", "java_import_external")
load("@io_bazel_rules_closure//closure:defs.bzl", "filegroup_external")
-def _extract_version_number(bazel_version):
- """Extracts the semantic version number from a version string
-
- Args:
- bazel_version: the version string that begins with the semantic version
- e.g. "1.2.3rc1 abc1234" where "abc1234" is a commit hash.
-
- Returns:
- The semantic version string, like "1.2.3".
- """
- for i in range(len(bazel_version)):
- c = bazel_version[i]
- if not (c.isdigit() or c == "."):
- return bazel_version[:i]
- return bazel_version
-
-# Parse the bazel version string from `native.bazel_version`.
-# e.g.
-# "0.10.0rc1 abc123d" => (0, 10, 0)
-# "0.3.0" => (0, 3, 0)
-def _parse_bazel_version(bazel_version):
- """Parses a version string into a 3-tuple of ints
-
- int tuples can be compared directly using binary operators (<, >).
-
- Args:
- bazel_version: the Bazel version string
-
- Returns:
- An int 3-tuple of a (major, minor, patch) version.
- """
-
- version = _extract_version_number(bazel_version)
- return tuple([int(n) for n in version.split(".")])
-
-def check_bazel_version_at_least(minimum_bazel_version):
- if "bazel_version" not in dir(native):
- fail("\nCurrent Bazel version is lower than 0.2.1, expected at least %s\n" % minimum_bazel_version)
- elif not native.bazel_version:
- print("\nCurrent Bazel is not a release version, cannot check for compatibility.")
- print("Make sure that you are running at least Bazel %s.\n" % minimum_bazel_version)
- return
-
- if _parse_bazel_version(native.bazel_version) < _parse_bazel_version(minimum_bazel_version):
- fail("\nCurrent Bazel version is {}, expected at least {}\n".format(
- native.bazel_version, minimum_bazel_version))
# If TensorFlow is linked as a submodule.
# path_prefix is no longer used.
# tf_repo_name is thought to be under consideration.
def tf_workspace(path_prefix="", tf_repo_name=""):
- # We must check the bazel version before trying to parse any other BUILD
- # files, in case the parsing of those build files depends on the bazel
- # version we require here.
- check_bazel_version_at_least("0.5.4")
+ # Note that we check the minimum bazel version in WORKSPACE.
clang6_configure(name="local_config_clang6")
+ cc_download_clang_toolchain(name="local_config_download_clang")
cuda_configure(name="local_config_cuda")
tensorrt_configure(name="local_config_tensorrt")
git_configure(name="local_config_git")
diff --git a/third_party/clang_toolchain/BUILD b/third_party/clang_toolchain/BUILD
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/third_party/clang_toolchain/BUILD
diff --git a/third_party/clang_toolchain/cc_configure_clang.bzl b/third_party/clang_toolchain/cc_configure_clang.bzl
new file mode 100644
index 0000000..1181110
--- /dev/null
+++ b/third_party/clang_toolchain/cc_configure_clang.bzl
@@ -0,0 +1,27 @@
+""" Downloads clang and configures the crosstool using bazel's autoconf."""
+
+load("@bazel_tools//tools/cpp:cc_configure.bzl", "cc_autoconf_impl")
+load(":download_clang.bzl", "download_clang")
+
+_TF_DOWNLOAD_CLANG = "TF_DOWNLOAD_CLANG"
+_TF_NEED_CUDA = "TF_NEED_CUDA"
+
+def _cc_clang_autoconf(repo_ctx):
+ if repo_ctx.os.environ.get(_TF_DOWNLOAD_CLANG) != "1":
+ return
+ if repo_ctx.os.environ.get(_TF_NEED_CUDA) == "1":
+ # Clang is handled separately for CUDA configs.
+ # See cuda_configure.bzl for more details.
+ return
+
+ download_clang(repo_ctx, out_folder='extra_tools')
+ overriden_tools = {'gcc': 'extra_tools/bin/clang'}
+ cc_autoconf_impl(repo_ctx, overriden_tools)
+
+cc_download_clang_toolchain = repository_rule(
+ environ = [
+ _TF_DOWNLOAD_CLANG,
+ _TF_NEED_CUDA,
+ ],
+ implementation = _cc_clang_autoconf,
+)
diff --git a/third_party/gpus/download_clang.bzl b/third_party/clang_toolchain/download_clang.bzl
similarity index 100%
rename from third_party/gpus/download_clang.bzl
rename to third_party/clang_toolchain/download_clang.bzl
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 6c9c128..ede7e31 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -96,7 +96,7 @@
"share/cuda/",
]
-load(":download_clang.bzl", "download_clang")
+load("//third_party/clang_toolchain:download_clang.bzl", "download_clang")
# TODO(dzc): Once these functions have been factored out of Bazel's
# cc_configure.bzl, load them from @bazel_tools instead.
diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD
index 752a0d8..68f24aa 100644
--- a/third_party/mkl_dnn/mkldnn.BUILD
+++ b/third_party/mkl_dnn/mkldnn.BUILD
@@ -4,7 +4,7 @@
name = "clang_linux_x86_64",
values = {
"cpu": "k8",
- "define": "using_cuda_clang=true",
+ "define": "using_clang=true",
},
)
diff --git a/tools/bazel.rc b/tools/bazel.rc
index 8b8c717..1c1e6af 100644
--- a/tools/bazel.rc
+++ b/tools/bazel.rc
@@ -27,11 +27,14 @@
build:mkl --define=using_mkl=true
build:mkl -c opt
+build:download_clang --crosstool_top=@local_config_download_clang//:toolchain
+build:download_clang --define=using_clang=true
+
build:cuda --crosstool_top=@local_config_cuda//crosstool:toolchain
build:cuda --define=using_cuda=true --define=using_cuda_nvcc=true
build:cuda_clang --crosstool_top=@local_config_cuda//crosstool:toolchain
-build:cuda_clang --define=using_cuda=true --define=using_cuda_clang=true
+build:cuda_clang --define=using_cuda=true --define=using_cuda_clang=true --define=using_clang=true
build:win-cuda --define=using_cuda=true --define=using_cuda_nvcc=true