Merge changes from github.
END_PUBLIC
---
Commit b30ce4714 authored by James Qin<jamesqin@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Revamp CudnnRNN Saveables
1. Use a lossy way to save/restore cudnn biases during checkpointing.
Cudnn uses 2 biases each gate for all RNNs while tf uses one. To allow cudnn checkpoints
to be compatible with both Cudnn and platform-independent impls, previously both
individual bias and summed biases each gate were stored.
The new way only stores the bias sum for each gate, and split it half-half when
restoring from a cudnn graph. Doing this does not cause problems since RNNs do not use
weight-decay to regularize.
2. Use inheritance instead of branching
* Split RNNParamsSaveable to 1 base class and 4 subclasses.
* Extract common routines and only overwrite rnn-type-specific pieces in subclasses.
PiperOrigin-RevId: 166413989
---
Commit ebc421daf authored by Alan Yee<alyee@ucsd.edu>
Committed by Jonathan Hseu<vomjom@vomjom.net>:
Update documentation for contrib (#12424)
* Update __init__.py
Remove ## for standardization of api docs
* Create README.md
Add README to define this directory's purpose
* Update __init.py
Markdown styling does not show up well in api docs
* Update README.md
Add short mention of describing what to deprecate
* Update README.md
Capitalize title
* Update README.md
Revert README change
* Delete README.md
---
Commit fd295394d authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Use latest version of nsync library, which now allows use of cmake on MacOS.
PiperOrigin-RevId: 166411437
---
Commit 587d728e0 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[XLA] Refactor reduce-precision-insertion filters, add several more options.
In particular, this adds the ability to add reduce-precision operations after fusion nodes based on the contents of those fusion nodes, and the ability to filter operations based on the "op_name" metadata.
PiperOrigin-RevId: 166408392
---
Commit 3142f8ef5 authored by Ali Yahya<alive@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Steps toward making ResourceVariables compatible with Eager.
This change forces the value of the reuse flag in variable scopes to be tf.AUTO_REUSE when in Eager mode.
This change also adds comprehensive Eager tests for ResourceVariable.
PiperOrigin-RevId: 166408161
---
Commit b2ce45150 authored by Igor Ganichev<iga@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Make Graph::IsValidNode public
It can be reimplemented with existing public APIs, but instead of doing so,
making this one public seems better.
PiperOrigin-RevId: 166407897
---
Commit 0a2f40e92 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[XLA::CPU] Fix HLO profiling in parallel CPU backend.
PiperOrigin-RevId: 166400211
---
Commit c4a58e3fd authored by Yao Zhang<yaozhang@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Identify frame ids for all nodes in a graph.
PiperOrigin-RevId: 166397615
---
Commit 989713f26 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
BEGIN_PUBLIC
Automated g4 rollback of changelist 166294015
PiperOrigin-RevId: 166521502
diff --git a/configure.py b/configure.py
index a0fbcda..1a0f71e 100644
--- a/configure.py
+++ b/configure.py
@@ -25,6 +25,11 @@
import subprocess
import sys
+try:
+ from shutil import which
+except ImportError:
+ from distutils.spawn import find_executable as which
+
_TF_BAZELRC = '.tf_configure.bazelrc'
_DEFAULT_CUDA_VERSION = '8.0'
_DEFAULT_CUDNN_VERSION = '6'
@@ -53,6 +58,10 @@
return platform.machine() == 'ppc64le'
+def is_cygwin():
+ return platform.system().startswith('CYGWIN_NT')
+
+
def get_input(question):
try:
try:
@@ -121,13 +130,20 @@
write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var)))
-def run_shell(cmd):
- return subprocess.check_output(cmd, shell=True).decode('UTF-8').strip()
+def run_shell(cmd, allow_non_zero=False):
+ if allow_non_zero:
+ try:
+ output = subprocess.check_output(cmd)
+ except subprocess.CalledProcessError as e:
+ output = e.output
+ else:
+ output = subprocess.check_output(cmd)
+ return output.decode('UTF-8').strip()
def cygpath(path):
"""Convert path from posix to windows."""
- return run_shell('cygpath -m "%s"' % path)
+ return run_shell(['cygpath', '-m', path])
def get_python_path(environ_cp, python_bin_path):
@@ -136,20 +152,14 @@
if environ_cp.get('PYTHONPATH'):
python_paths = environ_cp.get('PYTHONPATH').split(':')
try:
- check_input = [
- python_bin_path, '-c',
- 'import site; print("\\n".join(site.getsitepackages()))'
- ]
- library_paths = subprocess.check_output(check_input).decode(
- 'UTF-8').strip().split('\n')
+ library_paths = run_shell(
+ [python_bin_path, '-c',
+ 'import site; print("\\n".join(site.getsitepackages()))']).split("\n")
except subprocess.CalledProcessError:
- check_input = [
- python_bin_path, '-c', 'from distutils.sysconfig import get_python_lib;'
- + 'print(get_python_lib())'
- ]
- library_paths = [
- subprocess.check_output(check_input).decode('UTF-8').strip()
- ]
+ library_paths = [run_shell(
+ [python_bin_path, '-c',
+ 'from distutils.sysconfig import get_python_lib;'
+ 'print(get_python_lib())'])]
all_paths = set(python_paths + library_paths)
@@ -162,8 +172,7 @@
def get_python_major_version(python_bin_path):
"""Get the python major version."""
- check_input = [python_bin_path, '-c', 'import sys; print(sys.version[0])']
- return subprocess.check_output(check_input).decode('UTF-8').strip()
+ return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])'])
def setup_python(environ_cp, bazel_version):
@@ -177,8 +186,8 @@
environ_cp, 'PYTHON_BIN_PATH', ask_python_bin_path,
default_python_bin_path)
# Check if the path is valid
- if (os.path.isfile(python_bin_path) and os.access(
- python_bin_path, os.X_OK)) or (os.path.isdir(python_bin_path)):
+ if os.path.isfile(python_bin_path) and os.access(
+ python_bin_path, os.X_OK):
break
elif not os.path.exists(python_bin_path):
print('Invalid python path: %s cannot be found.' % python_bin_path)
@@ -187,7 +196,7 @@
environ_cp['PYTHON_BIN_PATH'] = ''
# Convert python path to Windows style before checking lib and version
- if is_windows():
+ if is_cygwin():
python_bin_path = cygpath(python_bin_path)
# Get PYTHON_LIB_PATH
@@ -197,12 +206,12 @@
if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1':
python_lib_path = python_lib_paths[0]
else:
- print('Found possible Python library paths:\n%s' %
- '\n'.join(python_lib_paths))
+ print('Found possible Python library paths:\n %s' %
+ '\n '.join(python_lib_paths))
default_python_lib_path = python_lib_paths[0]
python_lib_path = get_input(
- 'Please input the desired Python library path to use. Default is %s'
- % python_lib_paths[0])
+ 'Please input the desired Python library path to use. '
+ 'Default is [%s]\n' % python_lib_paths[0])
if not python_lib_path:
python_lib_path = default_python_lib_path
environ_cp['PYTHON_LIB_PATH'] = python_lib_path
@@ -210,7 +219,7 @@
python_major_version = get_python_major_version(python_bin_path)
# Convert python path to Windows style before writing into bazel.rc
- if is_windows():
+ if is_cygwin():
python_lib_path = cygpath(python_lib_path)
# Set-up env variables used by python_configure.bzl
@@ -432,11 +441,10 @@
Returns:
The bazel version detected.
"""
- try:
- curr_version = run_shell('bazel --batch version')
- except subprocess.CalledProcessError:
+ if which('bazel') is None:
print('Cannot find bazel. Please install bazel.')
sys.exit(0)
+ curr_version = run_shell(['bazel', '--batch', 'version'])
for line in curr_version.split('\n'):
if 'Build label: ' in line:
@@ -529,7 +537,7 @@
def set_clang_cuda_compiler_path(environ_cp):
"""Set CLANG_CUDA_COMPILER_PATH."""
- default_clang_path = run_shell('which clang || true')
+ default_clang_path = which('clang') or ''
ask_clang_path = ('Please specify which clang should be used as device and '
'host compiler. [Default is %s]: ') % default_clang_path
@@ -552,12 +560,12 @@
def set_gcc_host_compiler_path(environ_cp):
"""Set GCC_HOST_COMPILER_PATH."""
- default_gcc_host_compiler_path = run_shell('which gcc || true')
+ default_gcc_host_compiler_path = which('gcc') or ''
cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH')
if os.path.islink(cuda_bin_symlink):
# os.readlink is only available in linux
- default_gcc_host_compiler_path = run_shell('readlink %s' % cuda_bin_symlink)
+ default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink)
ask_gcc_path = (
'Please specify which gcc should be used by nvcc as the '
@@ -592,7 +600,7 @@
# Find out where the CUDA toolkit is installed
default_cuda_path = _DEFAULT_CUDA_PATH
- if is_windows():
+ if is_cygwin():
default_cuda_path = cygpath(
environ_cp.get('CUDA_PATH', _DEFAULT_CUDA_PATH_WIN))
elif is_linux():
@@ -633,7 +641,7 @@
def set_tf_cunn_version(environ_cp):
"""Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION."""
ask_cudnn_version = (
- '"Please specify the cuDNN version you want to use. '
+ 'Please specify the cuDNN version you want to use. '
'[Leave empty to default to cuDNN %s.0]: ') % _DEFAULT_CUDNN_VERSION
while True:
@@ -652,7 +660,7 @@
# unusable. Going through one more level of expansion to handle that.
cudnn_install_path = os.path.realpath(
os.path.expanduser(cudnn_install_path))
- if is_windows():
+ if is_cygwin():
cudnn_install_path = cygpath(cudnn_install_path)
if is_windows():
@@ -674,12 +682,10 @@
# Try another alternative for Linux
if is_linux():
- if subprocess.call(['which', 'ldconfig']):
- ldconfig_bin = '/sbin/ldconfig'
- else:
- ldconfig_bin = 'ldconfig'
- cudnn_path_from_ldconfig = run_shell(
- r'%s -p | sed -n "s/.*libcudnn.so .* => \(.*\)/\\1/p"' % ldconfig_bin)
+ ldconfig_bin = which('ldconfig') or '/sbin/ldconfig'
+ cudnn_path_from_ldconfig = run_shell([ldconfig_bin, '-p'])
+ cudnn_path_from_ldconfig = re.search('.*libcudnn.so .* => (.*)',
+ cudnn_path_from_ldconfig).group(1)
if os.path.exists('%s.%s' % (cudnn_path_from_ldconfig, tf_cudnn_version)):
cudnn_install_path = os.path.dirname(cudnn_path_from_ldconfig)
break
@@ -712,11 +718,15 @@
"""
device_query_bin = os.path.join(
environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery')
- cmd = (r'"%s" | grep "Capability" | grep -o "[0-9]*\.[0-9]*" | sed '
- '":a;{N;s/\\n/,/};ba"') % device_query_bin
- try:
- output = run_shell(cmd)
- except subprocess.CalledProcessError:
+ if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK):
+ try:
+ output = run_shell(device_query_bin).split('\n')
+ pattern = re.compile('[0-9]*\\.[0-9]*')
+ output = [pattern.search(x) for x in output if 'Capability' in x]
+ output = ','.join(x.group() for x in output if x is not None)
+ except subprocess.CalledProcessError:
+ output = ''
+ else:
output = ''
return output
@@ -797,7 +807,7 @@
def set_host_cxx_compiler(environ_cp):
"""Set HOST_CXX_COMPILER."""
- default_cxx_host_compiler = run_shell('which g++ || true')
+ default_cxx_host_compiler = which('g++') or ''
ask_cxx_host_compiler = (
'Please specify which C++ compiler should be used as'
' the host C++ compiler. [Default is %s]: ') % default_cxx_host_compiler
@@ -820,7 +830,7 @@
def set_host_c_compiler(environ_cp):
"""Set HOST_C_COMPILER."""
- default_c_host_compiler = run_shell('which gcc || true')
+ default_c_host_compiler = which('gcc') or ''
ask_c_host_compiler = (
'Please specify which C compiler should be used as the'
' host C compiler. [Default is %s]: ') % default_c_host_compiler
@@ -874,9 +884,9 @@
def set_mpi_home(environ_cp):
"""Set MPI_HOME."""
- cmd = ('dirname $(dirname $(which mpirun)) || dirname $(dirname $(which '
- 'mpiexec)) || true')
- default_mpi_home = run_shell(cmd)
+ default_mpi_home = which('mpirun') or which('mpiexec') or ''
+ default_mpi_home = os.path.dirname(os.path.dirname(default_mpi_home))
+
ask_mpi_home = ('Please specify the MPI toolkit folder. [Default is %s]: '
) % default_mpi_home
while True: