A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | # ============================================================================== |
| 15 | """configure script to get build parameters from user.""" |
| 16 | |
| 17 | from __future__ import absolute_import |
| 18 | from __future__ import division |
| 19 | from __future__ import print_function |
| 20 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 21 | import argparse |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 22 | import errno |
| 23 | import os |
| 24 | import platform |
| 25 | import re |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 26 | import subprocess |
| 27 | import sys |
| 28 | |
Andrew Selle | c9885ea | 2017-11-06 09:37:03 -0800 | [diff] [blame] | 29 | # pylint: disable=g-import-not-at-top |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 30 | try: |
| 31 | from shutil import which |
| 32 | except ImportError: |
| 33 | from distutils.spawn import find_executable as which |
Andrew Selle | c9885ea | 2017-11-06 09:37:03 -0800 | [diff] [blame] | 34 | # pylint: enable=g-import-not-at-top |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 35 | |
Christian Sigg | 8dff6b9 | 2019-12-09 04:05:46 -0800 | [diff] [blame] | 36 | _DEFAULT_CUDA_VERSION = '10' |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 37 | _DEFAULT_CUDNN_VERSION = '7' |
Guangda Lai | faa93ac | 2019-10-23 15:07:12 -0700 | [diff] [blame] | 38 | _DEFAULT_TENSORRT_VERSION = '6' |
Smit Hinsu | fe7d1d9 | 2018-07-14 13:16:58 -0700 | [diff] [blame] | 39 | _DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 40 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 41 | _TF_OPENCL_VERSION = '1.2' |
| 42 | _DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp' |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 43 | _DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include' |
A. Unique TensorFlower | 82820ef | 2018-11-12 13:22:13 -0800 | [diff] [blame] | 44 | _SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15, 16, 17, 18] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 45 | |
| 46 | _DEFAULT_PROMPT_ASK_ATTEMPTS = 10 |
| 47 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 48 | _TF_BAZELRC_FILENAME = '.tf_configure.bazelrc' |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 49 | _TF_WORKSPACE_ROOT = '' |
| 50 | _TF_BAZELRC = '' |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 51 | _TF_CURRENT_BAZEL_VERSION = None |
Gunhan Gulsoy | aaea541 | 2019-11-29 22:08:00 -0800 | [diff] [blame] | 52 | _TF_MIN_BAZEL_VERSION = '1.0.0' |
A. Unique TensorFlower | a5f9bcd | 2019-11-12 07:08:54 -0800 | [diff] [blame] | 53 | _TF_MAX_BAZEL_VERSION = '1.1.0' |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 54 | |
Jason Furmanek | 7c23415 | 2018-09-26 04:44:12 +0000 | [diff] [blame] | 55 | NCCL_LIB_PATHS = [ |
TensorFlower Gardener | ea5c529 | 2018-10-01 17:28:26 -0700 | [diff] [blame] | 56 | 'lib64/', 'lib/powerpc64le-linux-gnu/', 'lib/x86_64-linux-gnu/', '' |
Jason Furmanek | 7c23415 | 2018-09-26 04:44:12 +0000 | [diff] [blame] | 57 | ] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 58 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 59 | # List of files to configure when building Bazel on Apple platforms. |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 60 | APPLE_BAZEL_FILES = [ |
A. Unique TensorFlower | 6a059c0 | 2019-04-05 14:28:27 -0700 | [diff] [blame] | 61 | 'tensorflow/lite/experimental/ios/BUILD', |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 62 | 'tensorflow/lite/experimental/objc/BUILD', |
YoungSeok Yoon | 7089aaa | 2019-11-11 21:36:17 -0800 | [diff] [blame] | 63 | 'tensorflow/lite/experimental/swift/BUILD', |
| 64 | 'tensorflow/lite/tools/benchmark/experimental/ios/BUILD' |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 65 | ] |
| 66 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 67 | # List of files to move when building for iOS. |
| 68 | IOS_FILES = [ |
| 69 | 'tensorflow/lite/experimental/objc/TensorFlowLiteObjC.podspec', |
| 70 | 'tensorflow/lite/experimental/swift/TensorFlowLiteSwift.podspec', |
| 71 | ] |
| 72 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 73 | |
| 74 | class UserInputError(Exception): |
| 75 | pass |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 76 | |
| 77 | |
| 78 | def is_windows(): |
| 79 | return platform.system() == 'Windows' |
| 80 | |
| 81 | |
| 82 | def is_linux(): |
| 83 | return platform.system() == 'Linux' |
| 84 | |
| 85 | |
| 86 | def is_macos(): |
| 87 | return platform.system() == 'Darwin' |
| 88 | |
| 89 | |
| 90 | def is_ppc64le(): |
| 91 | return platform.machine() == 'ppc64le' |
| 92 | |
| 93 | |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 94 | def is_cygwin(): |
| 95 | return platform.system().startswith('CYGWIN_NT') |
| 96 | |
| 97 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 98 | def get_input(question): |
| 99 | try: |
| 100 | try: |
| 101 | answer = raw_input(question) |
| 102 | except NameError: |
| 103 | answer = input(question) # pylint: disable=bad-builtin |
| 104 | except EOFError: |
| 105 | answer = '' |
| 106 | return answer |
| 107 | |
| 108 | |
| 109 | def symlink_force(target, link_name): |
| 110 | """Force symlink, equivalent of 'ln -sf'. |
| 111 | |
| 112 | Args: |
| 113 | target: items to link to. |
| 114 | link_name: name of the link. |
| 115 | """ |
| 116 | try: |
| 117 | os.symlink(target, link_name) |
| 118 | except OSError as e: |
| 119 | if e.errno == errno.EEXIST: |
| 120 | os.remove(link_name) |
| 121 | os.symlink(target, link_name) |
| 122 | else: |
| 123 | raise e |
| 124 | |
| 125 | |
| 126 | def sed_in_place(filename, old, new): |
| 127 | """Replace old string with new string in file. |
| 128 | |
| 129 | Args: |
| 130 | filename: string for filename. |
| 131 | old: string to replace. |
| 132 | new: new string to replace to. |
| 133 | """ |
| 134 | with open(filename, 'r') as f: |
| 135 | filedata = f.read() |
| 136 | newdata = filedata.replace(old, new) |
| 137 | with open(filename, 'w') as f: |
| 138 | f.write(newdata) |
| 139 | |
| 140 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 141 | def write_to_bazelrc(line): |
| 142 | with open(_TF_BAZELRC, 'a') as f: |
| 143 | f.write(line + '\n') |
| 144 | |
| 145 | |
| 146 | def write_action_env_to_bazelrc(var_name, var): |
| 147 | write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var))) |
| 148 | |
| 149 | |
Wallyss Lima | 7237104 | 2019-10-03 12:55:30 -0300 | [diff] [blame^] | 150 | def run_shell(cmd, allow_non_zero=False, stderr=None): |
| 151 | if stderr is None: |
| 152 | stderr = sys.stdout |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 153 | if allow_non_zero: |
| 154 | try: |
Wallyss Lima | 7237104 | 2019-10-03 12:55:30 -0300 | [diff] [blame^] | 155 | output = subprocess.check_output(cmd, stderr=stderr) |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 156 | except subprocess.CalledProcessError as e: |
| 157 | output = e.output |
| 158 | else: |
Wallyss Lima | 7237104 | 2019-10-03 12:55:30 -0300 | [diff] [blame^] | 159 | output = subprocess.check_output(cmd, stderr=stderr) |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 160 | return output.decode('UTF-8').strip() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 161 | |
| 162 | |
| 163 | def cygpath(path): |
| 164 | """Convert path from posix to windows.""" |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 165 | return os.path.abspath(path).replace('\\', '/') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 166 | |
| 167 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 168 | def get_python_path(environ_cp, python_bin_path): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 169 | """Get the python site package paths.""" |
| 170 | python_paths = [] |
| 171 | if environ_cp.get('PYTHONPATH'): |
| 172 | python_paths = environ_cp.get('PYTHONPATH').split(':') |
| 173 | try: |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 174 | library_paths = run_shell([ |
| 175 | python_bin_path, '-c', |
| 176 | 'import site; print("\\n".join(site.getsitepackages()))' |
| 177 | ]).split('\n') |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 178 | except subprocess.CalledProcessError: |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 179 | library_paths = [ |
| 180 | run_shell([ |
| 181 | python_bin_path, '-c', |
| 182 | 'from distutils.sysconfig import get_python_lib;' |
| 183 | 'print(get_python_lib())' |
| 184 | ]) |
| 185 | ] |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 186 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 187 | all_paths = set(python_paths + library_paths) |
| 188 | |
| 189 | paths = [] |
| 190 | for path in all_paths: |
| 191 | if os.path.isdir(path): |
| 192 | paths.append(path) |
| 193 | return paths |
| 194 | |
| 195 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 196 | def get_python_major_version(python_bin_path): |
| 197 | """Get the python major version.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 198 | return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])']) |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 199 | |
| 200 | |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 201 | def setup_python(environ_cp): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 202 | """Setup python related env variables.""" |
| 203 | # Get PYTHON_BIN_PATH, default is the current running python. |
| 204 | default_python_bin_path = sys.executable |
| 205 | ask_python_bin_path = ('Please specify the location of python. [Default is ' |
| 206 | '%s]: ') % default_python_bin_path |
| 207 | while True: |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 208 | python_bin_path = get_from_env_or_user_or_default(environ_cp, |
| 209 | 'PYTHON_BIN_PATH', |
| 210 | ask_python_bin_path, |
| 211 | default_python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 212 | # Check if the path is valid |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 213 | if os.path.isfile(python_bin_path) and os.access(python_bin_path, os.X_OK): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 214 | break |
| 215 | elif not os.path.exists(python_bin_path): |
| 216 | print('Invalid python path: %s cannot be found.' % python_bin_path) |
| 217 | else: |
| 218 | print('%s is not executable. Is it the python binary?' % python_bin_path) |
| 219 | environ_cp['PYTHON_BIN_PATH'] = '' |
| 220 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 221 | # Convert python path to Windows style before checking lib and version |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 222 | if is_windows() or is_cygwin(): |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 223 | python_bin_path = cygpath(python_bin_path) |
| 224 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 225 | # Get PYTHON_LIB_PATH |
| 226 | python_lib_path = environ_cp.get('PYTHON_LIB_PATH') |
| 227 | if not python_lib_path: |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 228 | python_lib_paths = get_python_path(environ_cp, python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 229 | if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1': |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 230 | python_lib_path = python_lib_paths[0] |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 231 | else: |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 232 | print('Found possible Python library paths:\n %s' % |
| 233 | '\n '.join(python_lib_paths)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 234 | default_python_lib_path = python_lib_paths[0] |
| 235 | python_lib_path = get_input( |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 236 | 'Please input the desired Python library path to use. ' |
| 237 | 'Default is [%s]\n' % python_lib_paths[0]) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 238 | if not python_lib_path: |
| 239 | python_lib_path = default_python_lib_path |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 240 | environ_cp['PYTHON_LIB_PATH'] = python_lib_path |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 241 | |
A. Unique TensorFlower | cfccbdb | 2019-06-24 11:01:58 -0700 | [diff] [blame] | 242 | python_major_version = get_python_major_version(python_bin_path) |
| 243 | if python_major_version == '2': |
| 244 | write_to_bazelrc('build --host_force_python=PY2') |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 245 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 246 | # Convert python path to Windows style before writing into bazel.rc |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 247 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 248 | python_lib_path = cygpath(python_lib_path) |
| 249 | |
| 250 | # Set-up env variables used by python_configure.bzl |
| 251 | write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path) |
| 252 | write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path) |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 253 | write_to_bazelrc('build --python_path=\"%s"' % python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 254 | environ_cp['PYTHON_BIN_PATH'] = python_bin_path |
| 255 | |
William D. Irons | dcc76a5 | 2018-11-20 10:35:18 -0600 | [diff] [blame] | 256 | # If choosen python_lib_path is from a path specified in the PYTHONPATH |
| 257 | # variable, need to tell bazel to include PYTHONPATH |
| 258 | if environ_cp.get('PYTHONPATH'): |
| 259 | python_paths = environ_cp.get('PYTHONPATH').split(':') |
| 260 | if python_lib_path in python_paths: |
TensorFlower Gardener | 968cd18 | 2018-11-28 11:33:16 -0800 | [diff] [blame] | 261 | write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH')) |
William D. Irons | dcc76a5 | 2018-11-20 10:35:18 -0600 | [diff] [blame] | 262 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 263 | # Write tools/python_bin_path.sh |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 264 | with open( |
| 265 | os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'), |
| 266 | 'w') as f: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 267 | f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path) |
| 268 | |
| 269 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 270 | def reset_tf_configure_bazelrc(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 271 | """Reset file that contains customized config settings.""" |
| 272 | open(_TF_BAZELRC, 'w').close() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 273 | |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 274 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 275 | def cleanup_makefile(): |
| 276 | """Delete any leftover BUILD files from the Makefile build. |
| 277 | |
| 278 | These files could interfere with Bazel parsing. |
| 279 | """ |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 280 | makefile_download_dir = os.path.join(_TF_WORKSPACE_ROOT, 'tensorflow', |
| 281 | 'contrib', 'makefile', 'downloads') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 282 | if os.path.isdir(makefile_download_dir): |
| 283 | for root, _, filenames in os.walk(makefile_download_dir): |
| 284 | for f in filenames: |
| 285 | if f.endswith('BUILD'): |
| 286 | os.remove(os.path.join(root, f)) |
| 287 | |
| 288 | |
| 289 | def get_var(environ_cp, |
| 290 | var_name, |
| 291 | query_item, |
| 292 | enabled_by_default, |
| 293 | question=None, |
| 294 | yes_reply=None, |
| 295 | no_reply=None): |
| 296 | """Get boolean input from user. |
| 297 | |
| 298 | If var_name is not set in env, ask user to enable query_item or not. If the |
| 299 | response is empty, use the default. |
| 300 | |
| 301 | Args: |
| 302 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 303 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
| 304 | query_item: string for feature related to the variable, e.g. "CUDA for |
| 305 | Nvidia GPUs". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 306 | enabled_by_default: boolean for default behavior. |
| 307 | question: optional string for how to ask for user input. |
Michael Case | d90054e | 2018-02-07 14:36:00 -0800 | [diff] [blame] | 308 | yes_reply: optional string for reply when feature is enabled. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 309 | no_reply: optional string for reply when feature is disabled. |
| 310 | |
| 311 | Returns: |
| 312 | boolean value of the variable. |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 313 | |
| 314 | Raises: |
| 315 | UserInputError: if an environment variable is set, but it cannot be |
| 316 | interpreted as a boolean indicator, assume that the user has made a |
| 317 | scripting error, and will continue to provide invalid input. |
| 318 | Raise the error to avoid infinitely looping. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 319 | """ |
| 320 | if not question: |
| 321 | question = 'Do you wish to build TensorFlow with %s support?' % query_item |
| 322 | if not yes_reply: |
| 323 | yes_reply = '%s support will be enabled for TensorFlow.' % query_item |
| 324 | if not no_reply: |
| 325 | no_reply = 'No %s' % yes_reply |
| 326 | |
| 327 | yes_reply += '\n' |
| 328 | no_reply += '\n' |
| 329 | |
| 330 | if enabled_by_default: |
| 331 | question += ' [Y/n]: ' |
| 332 | else: |
| 333 | question += ' [y/N]: ' |
| 334 | |
| 335 | var = environ_cp.get(var_name) |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 336 | if var is not None: |
| 337 | var_content = var.strip().lower() |
| 338 | true_strings = ('1', 't', 'true', 'y', 'yes') |
| 339 | false_strings = ('0', 'f', 'false', 'n', 'no') |
| 340 | if var_content in true_strings: |
| 341 | var = True |
| 342 | elif var_content in false_strings: |
| 343 | var = False |
| 344 | else: |
| 345 | raise UserInputError( |
| 346 | 'Environment variable %s must be set as a boolean indicator.\n' |
| 347 | 'The following are accepted as TRUE : %s.\n' |
| 348 | 'The following are accepted as FALSE: %s.\n' |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 349 | 'Current value is %s.' % |
| 350 | (var_name, ', '.join(true_strings), ', '.join(false_strings), var)) |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 351 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 352 | while var is None: |
| 353 | user_input_origin = get_input(question) |
| 354 | user_input = user_input_origin.strip().lower() |
| 355 | if user_input == 'y': |
| 356 | print(yes_reply) |
| 357 | var = True |
| 358 | elif user_input == 'n': |
| 359 | print(no_reply) |
| 360 | var = False |
| 361 | elif not user_input: |
| 362 | if enabled_by_default: |
| 363 | print(yes_reply) |
| 364 | var = True |
| 365 | else: |
| 366 | print(no_reply) |
| 367 | var = False |
| 368 | else: |
| 369 | print('Invalid selection: %s' % user_input_origin) |
| 370 | return var |
| 371 | |
| 372 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 373 | def set_build_var(environ_cp, |
| 374 | var_name, |
| 375 | query_item, |
| 376 | option_name, |
| 377 | enabled_by_default, |
| 378 | bazel_config_name=None): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 379 | """Set if query_item will be enabled for the build. |
| 380 | |
| 381 | Ask user if query_item will be enabled. Default is used if no input is given. |
| 382 | Set subprocess environment variable and write to .bazelrc if enabled. |
| 383 | |
| 384 | Args: |
| 385 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 386 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
| 387 | query_item: string for feature related to the variable, e.g. "CUDA for |
| 388 | Nvidia GPUs". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 389 | option_name: string for option to define in .bazelrc. |
| 390 | enabled_by_default: boolean for default behavior. |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 391 | bazel_config_name: Name for Bazel --config argument to enable build feature. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 392 | """ |
| 393 | |
| 394 | var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default))) |
| 395 | environ_cp[var_name] = var |
| 396 | if var == '1': |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 397 | write_to_bazelrc('build:%s --define %s=true' % |
| 398 | (bazel_config_name, option_name)) |
Yifei Feng | ec451f5 | 2018-10-05 12:53:50 -0700 | [diff] [blame] | 399 | write_to_bazelrc('build --config=%s' % bazel_config_name) |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 400 | elif bazel_config_name is not None: |
| 401 | # TODO(mikecase): Migrate all users of configure.py to use --config Bazel |
| 402 | # options and not to set build configs through environment variables. |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 403 | write_to_bazelrc('build:%s --define %s=true' % |
| 404 | (bazel_config_name, option_name)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 405 | |
| 406 | |
| 407 | def set_action_env_var(environ_cp, |
| 408 | var_name, |
| 409 | query_item, |
| 410 | enabled_by_default, |
| 411 | question=None, |
| 412 | yes_reply=None, |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 413 | no_reply=None, |
| 414 | bazel_config_name=None): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 415 | """Set boolean action_env variable. |
| 416 | |
| 417 | Ask user if query_item will be enabled. Default is used if no input is given. |
| 418 | Set environment variable and write to .bazelrc. |
| 419 | |
| 420 | Args: |
| 421 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 422 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
| 423 | query_item: string for feature related to the variable, e.g. "CUDA for |
| 424 | Nvidia GPUs". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 425 | enabled_by_default: boolean for default behavior. |
| 426 | question: optional string for how to ask for user input. |
Michael Case | d90054e | 2018-02-07 14:36:00 -0800 | [diff] [blame] | 427 | yes_reply: optional string for reply when feature is enabled. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 428 | no_reply: optional string for reply when feature is disabled. |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 429 | bazel_config_name: adding config to .bazelrc instead of action_env. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 430 | """ |
| 431 | var = int( |
| 432 | get_var(environ_cp, var_name, query_item, enabled_by_default, question, |
| 433 | yes_reply, no_reply)) |
| 434 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 435 | if not bazel_config_name: |
| 436 | write_action_env_to_bazelrc(var_name, var) |
| 437 | elif var: |
| 438 | write_to_bazelrc('build --config=%s' % bazel_config_name) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 439 | environ_cp[var_name] = str(var) |
| 440 | |
| 441 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 442 | def convert_version_to_int(version): |
| 443 | """Convert a version number to a integer that can be used to compare. |
| 444 | |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 445 | Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The |
| 446 | 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored. |
| 447 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 448 | Args: |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 449 | version: a version to be converted |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 450 | |
| 451 | Returns: |
| 452 | An integer if converted successfully, otherwise return None. |
| 453 | """ |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 454 | version = version.split('-')[0] |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 455 | version_segments = version.split('.') |
Austin Anderson | 87ea41d | 2019-04-04 10:03:50 -0700 | [diff] [blame] | 456 | # Treat "0.24" as "0.24.0" |
| 457 | if len(version_segments) == 2: |
| 458 | version_segments.append('0') |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 459 | for seg in version_segments: |
| 460 | if not seg.isdigit(): |
| 461 | return None |
| 462 | |
| 463 | version_str = ''.join(['%03d' % int(seg) for seg in version_segments]) |
| 464 | return int(version_str) |
| 465 | |
| 466 | |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 467 | def check_bazel_version(min_version, max_version): |
| 468 | """Check installed bazel version is between min_version and max_version. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 469 | |
| 470 | Args: |
Mihai Maruseac | 4db860f | 2019-04-19 08:52:10 -0700 | [diff] [blame] | 471 | min_version: string for minimum bazel version (must exist!). |
| 472 | max_version: string for maximum bazel version (must exist!). |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 473 | |
| 474 | Returns: |
| 475 | The bazel version detected. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 476 | """ |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 477 | if which('bazel') is None: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 478 | print('Cannot find bazel. Please install bazel.') |
| 479 | sys.exit(0) |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 480 | curr_version = run_shell( |
| 481 | ['bazel', '--batch', '--bazelrc=/dev/null', 'version']) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 482 | |
| 483 | for line in curr_version.split('\n'): |
| 484 | if 'Build label: ' in line: |
| 485 | curr_version = line.split('Build label: ')[1] |
| 486 | break |
| 487 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 488 | min_version_int = convert_version_to_int(min_version) |
| 489 | curr_version_int = convert_version_to_int(curr_version) |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 490 | max_version_int = convert_version_to_int(max_version) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 491 | |
| 492 | # Check if current bazel version can be detected properly. |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 493 | if not curr_version_int: |
| 494 | print('WARNING: current bazel installation is not a release version.') |
| 495 | print('Make sure you are running at least bazel %s' % min_version) |
| 496 | return curr_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 497 | |
Michael Case | d94271a | 2017-08-22 17:26:52 -0700 | [diff] [blame] | 498 | print('You have bazel %s installed.' % curr_version) |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 499 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 500 | if curr_version_int < min_version_int: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 501 | print('Please upgrade your bazel installation to version %s or higher to ' |
| 502 | 'build TensorFlow!' % min_version) |
Jason Zaman | b41761c | 2018-10-14 11:28:53 +0800 | [diff] [blame] | 503 | sys.exit(1) |
TensorFlower Gardener | 78c246b | 2018-12-13 12:37:42 -0800 | [diff] [blame] | 504 | if (curr_version_int > max_version_int and |
| 505 | 'TF_IGNORE_MAX_BAZEL_VERSION' not in os.environ): |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 506 | print('Please downgrade your bazel installation to version %s or lower to ' |
Mihai Maruseac | e0963c4 | 2018-12-20 14:27:40 -0800 | [diff] [blame] | 507 | 'build TensorFlow! To downgrade: download the installer for the old ' |
| 508 | 'version (from https://github.com/bazelbuild/bazel/releases) then ' |
| 509 | 'run the installer.' % max_version) |
Jason Zaman | b41761c | 2018-10-14 11:28:53 +0800 | [diff] [blame] | 510 | sys.exit(1) |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 511 | return curr_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 512 | |
| 513 | |
| 514 | def set_cc_opt_flags(environ_cp): |
| 515 | """Set up architecture-dependent optimization flags. |
| 516 | |
| 517 | Also append CC optimization flags to bazel.rc.. |
| 518 | |
| 519 | Args: |
| 520 | environ_cp: copy of the os.environ. |
| 521 | """ |
| 522 | if is_ppc64le(): |
| 523 | # gcc on ppc64le does not support -march, use mcpu instead |
| 524 | default_cc_opt_flags = '-mcpu=native' |
A. Unique TensorFlower | 1bba94a | 2018-04-04 15:45:20 -0700 | [diff] [blame] | 525 | elif is_windows(): |
| 526 | default_cc_opt_flags = '/arch:AVX' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 527 | else: |
Justin Lebar | 9ef04f5 | 2018-10-10 18:52:45 -0700 | [diff] [blame] | 528 | default_cc_opt_flags = '-march=native -Wno-sign-compare' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 529 | question = ('Please specify optimization flags to use during compilation when' |
| 530 | ' bazel option "--config=opt" is specified [Default is %s]: ' |
| 531 | ) % default_cc_opt_flags |
| 532 | cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS', |
| 533 | question, default_cc_opt_flags) |
| 534 | for opt in cc_opt_flags.split(): |
Michael Case | 0017742 | 2017-11-10 13:14:03 -0800 | [diff] [blame] | 535 | write_to_bazelrc('build:opt --copt=%s' % opt) |
| 536 | # It should be safe on the same build host. |
A. Unique TensorFlower | 1bba94a | 2018-04-04 15:45:20 -0700 | [diff] [blame] | 537 | if not is_ppc64le() and not is_windows(): |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 538 | write_to_bazelrc('build:opt --host_copt=-march=native') |
Michael Case | bb3355d | 2017-11-09 08:46:31 -0800 | [diff] [blame] | 539 | write_to_bazelrc('build:opt --define with_default_optimizations=true') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 540 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 541 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 542 | def set_tf_cuda_clang(environ_cp): |
| 543 | """set TF_CUDA_CLANG action_env. |
| 544 | |
| 545 | Args: |
| 546 | environ_cp: copy of the os.environ. |
| 547 | """ |
| 548 | question = 'Do you want to use clang as CUDA compiler?' |
| 549 | yes_reply = 'Clang will be used as CUDA compiler.' |
| 550 | no_reply = 'nvcc will be used as CUDA compiler.' |
| 551 | set_action_env_var( |
| 552 | environ_cp, |
| 553 | 'TF_CUDA_CLANG', |
| 554 | None, |
| 555 | False, |
| 556 | question=question, |
| 557 | yes_reply=yes_reply, |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 558 | no_reply=no_reply, |
| 559 | bazel_config_name='cuda_clang') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 560 | |
| 561 | |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 562 | def set_tf_download_clang(environ_cp): |
| 563 | """Set TF_DOWNLOAD_CLANG action_env.""" |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 564 | question = 'Do you wish to download a fresh release of clang? (Experimental)' |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 565 | yes_reply = 'Clang will be downloaded and used to compile tensorflow.' |
| 566 | no_reply = 'Clang will not be downloaded.' |
| 567 | set_action_env_var( |
| 568 | environ_cp, |
| 569 | 'TF_DOWNLOAD_CLANG', |
| 570 | None, |
| 571 | False, |
| 572 | question=question, |
| 573 | yes_reply=yes_reply, |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 574 | no_reply=no_reply, |
| 575 | bazel_config_name='download_clang') |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 576 | |
| 577 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 578 | def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var, |
| 579 | var_default): |
| 580 | """Get var_name either from env, or user or default. |
| 581 | |
| 582 | If var_name has been set as environment variable, use the preset value, else |
| 583 | ask for user input. If no input is provided, the default is used. |
| 584 | |
| 585 | Args: |
| 586 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 587 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 588 | ask_for_var: string for how to ask for user input. |
| 589 | var_default: default value string. |
| 590 | |
| 591 | Returns: |
| 592 | string value for var_name |
| 593 | """ |
| 594 | var = environ_cp.get(var_name) |
| 595 | if not var: |
| 596 | var = get_input(ask_for_var) |
Michael Case | d94271a | 2017-08-22 17:26:52 -0700 | [diff] [blame] | 597 | print('\n') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 598 | if not var: |
| 599 | var = var_default |
| 600 | return var |
| 601 | |
| 602 | |
| 603 | def set_clang_cuda_compiler_path(environ_cp): |
| 604 | """Set CLANG_CUDA_COMPILER_PATH.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 605 | default_clang_path = which('clang') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 606 | ask_clang_path = ('Please specify which clang should be used as device and ' |
| 607 | 'host compiler. [Default is %s]: ') % default_clang_path |
| 608 | |
| 609 | while True: |
| 610 | clang_cuda_compiler_path = get_from_env_or_user_or_default( |
| 611 | environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path, |
| 612 | default_clang_path) |
| 613 | if os.path.exists(clang_cuda_compiler_path): |
| 614 | break |
| 615 | |
| 616 | # Reset and retry |
| 617 | print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path) |
| 618 | environ_cp['CLANG_CUDA_COMPILER_PATH'] = '' |
| 619 | |
| 620 | # Set CLANG_CUDA_COMPILER_PATH |
| 621 | environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path |
| 622 | write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH', |
| 623 | clang_cuda_compiler_path) |
| 624 | |
| 625 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 626 | def prompt_loop_or_load_from_env(environ_cp, |
| 627 | var_name, |
| 628 | var_default, |
| 629 | ask_for_var, |
| 630 | check_success, |
| 631 | error_msg, |
| 632 | suppress_default_error=False, |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 633 | resolve_symlinks=False, |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 634 | n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS): |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 635 | """Loop over user prompts for an ENV param until receiving a valid response. |
| 636 | |
| 637 | For the env param var_name, read from the environment or verify user input |
| 638 | until receiving valid input. When done, set var_name in the environ_cp to its |
| 639 | new value. |
| 640 | |
| 641 | Args: |
| 642 | environ_cp: (Dict) copy of the os.environ. |
| 643 | var_name: (String) string for name of environment variable, e.g. "TF_MYVAR". |
| 644 | var_default: (String) default value string. |
| 645 | ask_for_var: (String) string for how to ask for user input. |
| 646 | check_success: (Function) function that takes one argument and returns a |
| 647 | boolean. Should return True if the value provided is considered valid. May |
| 648 | contain a complex error message if error_msg does not provide enough |
| 649 | information. In that case, set suppress_default_error to True. |
| 650 | error_msg: (String) String with one and only one '%s'. Formatted with each |
| 651 | invalid response upon check_success(input) failure. |
| 652 | suppress_default_error: (Bool) Suppress the above error message in favor of |
| 653 | one from the check_success function. |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 654 | resolve_symlinks: (Bool) Translate symbolic links into the real filepath. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 655 | n_ask_attempts: (Integer) Number of times to query for valid input before |
| 656 | raising an error and quitting. |
| 657 | |
| 658 | Returns: |
| 659 | [String] The value of var_name after querying for input. |
| 660 | |
| 661 | Raises: |
| 662 | UserInputError: if a query has been attempted n_ask_attempts times without |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 663 | success, assume that the user has made a scripting error, and will |
| 664 | continue to provide invalid input. Raise the error to avoid infinitely |
| 665 | looping. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 666 | """ |
| 667 | default = environ_cp.get(var_name) or var_default |
| 668 | full_query = '%s [Default is %s]: ' % ( |
| 669 | ask_for_var, |
| 670 | default, |
| 671 | ) |
| 672 | |
| 673 | for _ in range(n_ask_attempts): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 674 | val = get_from_env_or_user_or_default(environ_cp, var_name, full_query, |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 675 | default) |
| 676 | if check_success(val): |
| 677 | break |
| 678 | if not suppress_default_error: |
| 679 | print(error_msg % val) |
| 680 | environ_cp[var_name] = '' |
| 681 | else: |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 682 | raise UserInputError('Invalid %s setting was provided %d times in a row. ' |
| 683 | 'Assuming to be a scripting mistake.' % |
| 684 | (var_name, n_ask_attempts)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 685 | |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 686 | if resolve_symlinks and os.path.islink(val): |
| 687 | val = os.path.realpath(val) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 688 | environ_cp[var_name] = val |
| 689 | return val |
| 690 | |
| 691 | |
| 692 | def create_android_ndk_rule(environ_cp): |
| 693 | """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule.""" |
| 694 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 695 | default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' % |
| 696 | environ_cp['APPDATA']) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 697 | elif is_macos(): |
| 698 | default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 699 | else: |
| 700 | default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 701 | |
| 702 | def valid_ndk_path(path): |
| 703 | return (os.path.exists(path) and |
| 704 | os.path.exists(os.path.join(path, 'source.properties'))) |
| 705 | |
| 706 | android_ndk_home_path = prompt_loop_or_load_from_env( |
| 707 | environ_cp, |
| 708 | var_name='ANDROID_NDK_HOME', |
| 709 | var_default=default_ndk_path, |
| 710 | ask_for_var='Please specify the home path of the Android NDK to use.', |
| 711 | check_success=valid_ndk_path, |
| 712 | error_msg=('The path %s or its child file "source.properties" ' |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 713 | 'does not exist.')) |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 714 | write_action_env_to_bazelrc('ANDROID_NDK_HOME', android_ndk_home_path) |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 715 | write_action_env_to_bazelrc( |
| 716 | 'ANDROID_NDK_API_LEVEL', |
| 717 | get_ndk_api_level(environ_cp, android_ndk_home_path)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 718 | |
| 719 | |
| 720 | def create_android_sdk_rule(environ_cp): |
| 721 | """Set Android variables and write Android SDK WORKSPACE rule.""" |
| 722 | if is_windows() or is_cygwin(): |
| 723 | default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA']) |
| 724 | elif is_macos(): |
Shashi Shekhar | c0ff0cc | 2018-07-17 09:00:24 -0700 | [diff] [blame] | 725 | default_sdk_path = '%s/library/Android/Sdk' % environ_cp['HOME'] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 726 | else: |
| 727 | default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME'] |
| 728 | |
| 729 | def valid_sdk_path(path): |
| 730 | return (os.path.exists(path) and |
| 731 | os.path.exists(os.path.join(path, 'platforms')) and |
| 732 | os.path.exists(os.path.join(path, 'build-tools'))) |
| 733 | |
| 734 | android_sdk_home_path = prompt_loop_or_load_from_env( |
| 735 | environ_cp, |
| 736 | var_name='ANDROID_SDK_HOME', |
| 737 | var_default=default_sdk_path, |
| 738 | ask_for_var='Please specify the home path of the Android SDK to use.', |
| 739 | check_success=valid_sdk_path, |
| 740 | error_msg=('Either %s does not exist, or it does not contain the ' |
| 741 | 'subdirectories "platforms" and "build-tools".')) |
| 742 | |
| 743 | platforms = os.path.join(android_sdk_home_path, 'platforms') |
| 744 | api_levels = sorted(os.listdir(platforms)) |
| 745 | api_levels = [x.replace('android-', '') for x in api_levels] |
| 746 | |
| 747 | def valid_api_level(api_level): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 748 | return os.path.exists( |
| 749 | os.path.join(android_sdk_home_path, 'platforms', |
| 750 | 'android-' + api_level)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 751 | |
| 752 | android_api_level = prompt_loop_or_load_from_env( |
| 753 | environ_cp, |
| 754 | var_name='ANDROID_API_LEVEL', |
| 755 | var_default=api_levels[-1], |
| 756 | ask_for_var=('Please specify the Android SDK API level to use. ' |
| 757 | '[Available levels: %s]') % api_levels, |
| 758 | check_success=valid_api_level, |
| 759 | error_msg='Android-%s is not present in the SDK path.') |
| 760 | |
| 761 | build_tools = os.path.join(android_sdk_home_path, 'build-tools') |
| 762 | versions = sorted(os.listdir(build_tools)) |
| 763 | |
| 764 | def valid_build_tools(version): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 765 | return os.path.exists( |
| 766 | os.path.join(android_sdk_home_path, 'build-tools', version)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 767 | |
| 768 | android_build_tools_version = prompt_loop_or_load_from_env( |
| 769 | environ_cp, |
| 770 | var_name='ANDROID_BUILD_TOOLS_VERSION', |
| 771 | var_default=versions[-1], |
| 772 | ask_for_var=('Please specify an Android build tools version to use. ' |
| 773 | '[Available versions: %s]') % versions, |
| 774 | check_success=valid_build_tools, |
| 775 | error_msg=('The selected SDK does not have build-tools version %s ' |
| 776 | 'available.')) |
| 777 | |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 778 | write_action_env_to_bazelrc('ANDROID_BUILD_TOOLS_VERSION', |
| 779 | android_build_tools_version) |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 780 | write_action_env_to_bazelrc('ANDROID_SDK_API_LEVEL', android_api_level) |
| 781 | write_action_env_to_bazelrc('ANDROID_SDK_HOME', android_sdk_home_path) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 782 | |
| 783 | |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 784 | def get_ndk_api_level(environ_cp, android_ndk_home_path): |
| 785 | """Gets the appropriate NDK API level to use for the provided Android NDK path.""" |
| 786 | |
| 787 | # First check to see if we're using a blessed version of the NDK. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 788 | properties_path = '%s/source.properties' % android_ndk_home_path |
| 789 | if is_windows() or is_cygwin(): |
| 790 | properties_path = cygpath(properties_path) |
| 791 | with open(properties_path, 'r') as f: |
| 792 | filedata = f.read() |
| 793 | |
| 794 | revision = re.search(r'Pkg.Revision = (\d+)', filedata) |
| 795 | if revision: |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 796 | ndk_version = revision.group(1) |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 797 | else: |
| 798 | raise Exception('Unable to parse NDK revision.') |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 799 | if int(ndk_version) not in _SUPPORTED_ANDROID_NDK_VERSIONS: |
| 800 | print('WARNING: The NDK version in %s is %s, which is not ' |
| 801 | 'supported by Bazel (officially supported versions: %s). Please use ' |
| 802 | 'another version. Compiling Android targets may result in confusing ' |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 803 | 'errors.\n' % |
| 804 | (android_ndk_home_path, ndk_version, _SUPPORTED_ANDROID_NDK_VERSIONS)) |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 805 | |
| 806 | # Now grab the NDK API level to use. Note that this is different from the |
| 807 | # SDK API level, as the NDK API level is effectively the *min* target SDK |
| 808 | # version. |
| 809 | platforms = os.path.join(android_ndk_home_path, 'platforms') |
| 810 | api_levels = sorted(os.listdir(platforms)) |
| 811 | api_levels = [ |
| 812 | x.replace('android-', '') for x in api_levels if 'android-' in x |
| 813 | ] |
| 814 | |
| 815 | def valid_api_level(api_level): |
| 816 | return os.path.exists( |
| 817 | os.path.join(android_ndk_home_path, 'platforms', |
| 818 | 'android-' + api_level)) |
| 819 | |
| 820 | android_ndk_api_level = prompt_loop_or_load_from_env( |
| 821 | environ_cp, |
| 822 | var_name='ANDROID_NDK_API_LEVEL', |
Jared Duke | a1309f4 | 2019-11-15 17:09:35 -0800 | [diff] [blame] | 823 | var_default='21', # 21 is required for ARM64 support. |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 824 | ask_for_var=('Please specify the (min) Android NDK API level to use. ' |
| 825 | '[Available levels: %s]') % api_levels, |
| 826 | check_success=valid_api_level, |
| 827 | error_msg='Android-%s is not present in the NDK path.') |
| 828 | |
| 829 | return android_ndk_api_level |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 830 | |
| 831 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 832 | def set_gcc_host_compiler_path(environ_cp): |
| 833 | """Set GCC_HOST_COMPILER_PATH.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 834 | default_gcc_host_compiler_path = which('gcc') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 835 | cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH') |
| 836 | |
| 837 | if os.path.islink(cuda_bin_symlink): |
| 838 | # os.readlink is only available in linux |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 839 | default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 840 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 841 | gcc_host_compiler_path = prompt_loop_or_load_from_env( |
| 842 | environ_cp, |
| 843 | var_name='GCC_HOST_COMPILER_PATH', |
| 844 | var_default=default_gcc_host_compiler_path, |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 845 | ask_for_var='Please specify which gcc should be used by nvcc as the host compiler.', |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 846 | check_success=os.path.exists, |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 847 | resolve_symlinks=True, |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 848 | error_msg='Invalid gcc path. %s cannot be found.', |
| 849 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 850 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 851 | write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path) |
| 852 | |
| 853 | |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 854 | def reformat_version_sequence(version_str, sequence_count): |
| 855 | """Reformat the version string to have the given number of sequences. |
| 856 | |
| 857 | For example: |
| 858 | Given (7, 2) -> 7.0 |
| 859 | (7.0.1, 2) -> 7.0 |
| 860 | (5, 1) -> 5 |
| 861 | (5.0.3.2, 1) -> 5 |
| 862 | |
| 863 | Args: |
| 864 | version_str: String, the version string. |
| 865 | sequence_count: int, an integer. |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 866 | |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 867 | Returns: |
| 868 | string, reformatted version string. |
| 869 | """ |
| 870 | v = version_str.split('.') |
| 871 | if len(v) < sequence_count: |
| 872 | v = v + (['0'] * (sequence_count - len(v))) |
| 873 | |
| 874 | return '.'.join(v[:sequence_count]) |
| 875 | |
| 876 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 877 | def set_tf_cuda_paths(environ_cp): |
| 878 | """Set TF_CUDA_PATHS.""" |
| 879 | ask_cuda_paths = ( |
| 880 | 'Please specify the comma-separated list of base paths to look for CUDA ' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 881 | 'libraries and headers. [Leave empty to use the default]: ') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 882 | tf_cuda_paths = get_from_env_or_user_or_default(environ_cp, 'TF_CUDA_PATHS', |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 883 | ask_cuda_paths, '') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 884 | if tf_cuda_paths: |
| 885 | environ_cp['TF_CUDA_PATHS'] = tf_cuda_paths |
| 886 | |
| 887 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 888 | def set_tf_cuda_version(environ_cp): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 889 | """Set TF_CUDA_VERSION.""" |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 890 | ask_cuda_version = ( |
| 891 | 'Please specify the CUDA SDK version you want to use. ' |
| 892 | '[Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 893 | tf_cuda_version = get_from_env_or_user_or_default(environ_cp, |
| 894 | 'TF_CUDA_VERSION', |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 895 | ask_cuda_version, |
| 896 | _DEFAULT_CUDA_VERSION) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 897 | environ_cp['TF_CUDA_VERSION'] = tf_cuda_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 898 | |
| 899 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 900 | def set_tf_cudnn_version(environ_cp): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 901 | """Set TF_CUDNN_VERSION.""" |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 902 | ask_cudnn_version = ( |
| 903 | 'Please specify the cuDNN version you want to use. ' |
| 904 | '[Leave empty to default to cuDNN %s]: ') % _DEFAULT_CUDNN_VERSION |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 905 | tf_cudnn_version = get_from_env_or_user_or_default(environ_cp, |
| 906 | 'TF_CUDNN_VERSION', |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 907 | ask_cudnn_version, |
| 908 | _DEFAULT_CUDNN_VERSION) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 909 | environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 910 | |
| 911 | |
Mingxing Tan | 1e7b0e4 | 2018-06-28 19:13:20 -0700 | [diff] [blame] | 912 | def is_cuda_compatible(lib, cuda_ver, cudnn_ver): |
| 913 | """Check compatibility between given library and cudnn/cudart libraries.""" |
| 914 | ldd_bin = which('ldd') or '/usr/bin/ldd' |
| 915 | ldd_out = run_shell([ldd_bin, lib], True) |
| 916 | ldd_out = ldd_out.split(os.linesep) |
| 917 | cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$') |
| 918 | cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$') |
| 919 | cudnn = None |
| 920 | cudart = None |
| 921 | cudnn_ok = True # assume no cudnn dependency by default |
| 922 | cuda_ok = True # assume no cuda dependency by default |
| 923 | for line in ldd_out: |
| 924 | if 'libcudnn.so' in line: |
| 925 | cudnn = cudnn_pattern.search(line) |
| 926 | cudnn_ok = False |
| 927 | elif 'libcudart.so' in line: |
| 928 | cudart = cuda_pattern.search(line) |
| 929 | cuda_ok = False |
| 930 | if cudnn and len(cudnn.group(1)): |
| 931 | cudnn = convert_version_to_int(cudnn.group(1)) |
| 932 | if cudart and len(cudart.group(1)): |
| 933 | cudart = convert_version_to_int(cudart.group(1)) |
| 934 | if cudnn is not None: |
| 935 | cudnn_ok = (cudnn == cudnn_ver) |
| 936 | if cudart is not None: |
| 937 | cuda_ok = (cudart == cuda_ver) |
| 938 | return cudnn_ok and cuda_ok |
| 939 | |
| 940 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 941 | def set_tf_tensorrt_version(environ_cp): |
| 942 | """Set TF_TENSORRT_VERSION.""" |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 943 | if not is_linux(): |
| 944 | raise ValueError('Currently TensorRT is only supported on Linux platform.') |
| 945 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 946 | if not int(environ_cp.get('TF_NEED_TENSORRT', False)): |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 947 | return |
| 948 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 949 | ask_tensorrt_version = ( |
| 950 | 'Please specify the TensorRT version you want to use. ' |
Guangda Lai | faa93ac | 2019-10-23 15:07:12 -0700 | [diff] [blame] | 951 | '[Leave empty to default to TensorRT %s]: ') % _DEFAULT_TENSORRT_VERSION |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 952 | tf_tensorrt_version = get_from_env_or_user_or_default( |
| 953 | environ_cp, 'TF_TENSORRT_VERSION', ask_tensorrt_version, |
| 954 | _DEFAULT_TENSORRT_VERSION) |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 955 | environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 956 | |
| 957 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 958 | def set_tf_nccl_version(environ_cp): |
| 959 | """Set TF_NCCL_VERSION.""" |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 960 | if not is_linux(): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 961 | raise ValueError('Currently NCCL is only supported on Linux platform.') |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 962 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 963 | if 'TF_NCCL_VERSION' in environ_cp: |
| 964 | return |
| 965 | |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 966 | ask_nccl_version = ( |
A. Unique TensorFlower | 53faa31 | 2018-10-05 08:46:54 -0700 | [diff] [blame] | 967 | 'Please specify the locally installed NCCL version you want to use. ' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 968 | '[Leave empty to use http://github.com/nvidia/nccl]: ') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 969 | tf_nccl_version = get_from_env_or_user_or_default(environ_cp, |
| 970 | 'TF_NCCL_VERSION', |
| 971 | ask_nccl_version, '') |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 972 | environ_cp['TF_NCCL_VERSION'] = tf_nccl_version |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 973 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 974 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 975 | def get_native_cuda_compute_capabilities(environ_cp): |
| 976 | """Get native cuda compute capabilities. |
| 977 | |
| 978 | Args: |
| 979 | environ_cp: copy of the os.environ. |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 980 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 981 | Returns: |
| 982 | string of native cuda compute capabilities, separated by comma. |
| 983 | """ |
| 984 | device_query_bin = os.path.join( |
| 985 | environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery') |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 986 | if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK): |
| 987 | try: |
| 988 | output = run_shell(device_query_bin).split('\n') |
| 989 | pattern = re.compile('[0-9]*\\.[0-9]*') |
| 990 | output = [pattern.search(x) for x in output if 'Capability' in x] |
| 991 | output = ','.join(x.group() for x in output if x is not None) |
| 992 | except subprocess.CalledProcessError: |
| 993 | output = '' |
| 994 | else: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 995 | output = '' |
| 996 | return output |
| 997 | |
| 998 | |
| 999 | def set_tf_cuda_compute_capabilities(environ_cp): |
| 1000 | """Set TF_CUDA_COMPUTE_CAPABILITIES.""" |
| 1001 | while True: |
| 1002 | native_cuda_compute_capabilities = get_native_cuda_compute_capabilities( |
| 1003 | environ_cp) |
| 1004 | if not native_cuda_compute_capabilities: |
| 1005 | default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES |
| 1006 | else: |
| 1007 | default_cuda_compute_capabilities = native_cuda_compute_capabilities |
| 1008 | |
| 1009 | ask_cuda_compute_capabilities = ( |
| 1010 | 'Please specify a list of comma-separated ' |
P Sudeepam | 5209356 | 2019-02-17 17:34:01 +0530 | [diff] [blame] | 1011 | 'CUDA compute capabilities you want to ' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1012 | 'build with.\nYou can find the compute ' |
| 1013 | 'capability of your device at: ' |
| 1014 | 'https://developer.nvidia.com/cuda-gpus.\nPlease' |
| 1015 | ' note that each additional compute ' |
| 1016 | 'capability significantly increases your ' |
P Sudeepam | 5209356 | 2019-02-17 17:34:01 +0530 | [diff] [blame] | 1017 | 'build time and binary size, and that ' |
| 1018 | 'TensorFlow only supports compute ' |
P Sudeepam | 765ceda | 2019-02-17 17:39:08 +0530 | [diff] [blame] | 1019 | 'capabilities >= 3.5 [Default is: %s]: ' % |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1020 | default_cuda_compute_capabilities) |
| 1021 | tf_cuda_compute_capabilities = get_from_env_or_user_or_default( |
| 1022 | environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES', |
| 1023 | ask_cuda_compute_capabilities, default_cuda_compute_capabilities) |
| 1024 | # Check whether all capabilities from the input is valid |
| 1025 | all_valid = True |
Maciej | d0f5bc1 | 2018-04-30 22:30:58 -0500 | [diff] [blame] | 1026 | # Remove all whitespace characters before splitting the string |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 1027 | # that users may insert by accident, as this will result in error |
Maciej | d0f5bc1 | 2018-04-30 22:30:58 -0500 | [diff] [blame] | 1028 | tf_cuda_compute_capabilities = ''.join(tf_cuda_compute_capabilities.split()) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1029 | for compute_capability in tf_cuda_compute_capabilities.split(','): |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1030 | m = re.match('[0-9]+.[0-9]+', compute_capability) |
| 1031 | if not m: |
Austin Anderson | 32202dc | 2019-02-19 10:46:27 -0800 | [diff] [blame] | 1032 | print('Invalid compute capability: %s' % compute_capability) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1033 | all_valid = False |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1034 | else: |
P Sudeepam | 5209356 | 2019-02-17 17:34:01 +0530 | [diff] [blame] | 1035 | ver = float(m.group(0)) |
A. Unique TensorFlower | 8dc2d0e | 2019-03-12 01:41:05 -0700 | [diff] [blame] | 1036 | if ver < 3.0: |
| 1037 | print('ERROR: TensorFlow only supports CUDA compute capabilities 3.0 ' |
Austin Anderson | 32202dc | 2019-02-19 10:46:27 -0800 | [diff] [blame] | 1038 | 'and higher. Please re-specify the list of compute ' |
| 1039 | 'capabilities excluding version %s.' % ver) |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1040 | all_valid = False |
A. Unique TensorFlower | 8dc2d0e | 2019-03-12 01:41:05 -0700 | [diff] [blame] | 1041 | if ver < 3.5: |
| 1042 | print('WARNING: XLA does not support CUDA compute capabilities ' |
| 1043 | 'lower than 3.5. Disable XLA when running on older GPUs.') |
P Sudeepam | 765ceda | 2019-02-17 17:39:08 +0530 | [diff] [blame] | 1044 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1045 | if all_valid: |
| 1046 | break |
| 1047 | |
| 1048 | # Reset and Retry |
| 1049 | environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = '' |
| 1050 | |
| 1051 | # Set TF_CUDA_COMPUTE_CAPABILITIES |
| 1052 | environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities |
| 1053 | write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES', |
| 1054 | tf_cuda_compute_capabilities) |
| 1055 | |
| 1056 | |
| 1057 | def set_other_cuda_vars(environ_cp): |
| 1058 | """Set other CUDA related variables.""" |
A. Unique TensorFlower | ab39198 | 2018-07-11 04:52:49 -0700 | [diff] [blame] | 1059 | # If CUDA is enabled, always use GPU during build and test. |
| 1060 | if environ_cp.get('TF_CUDA_CLANG') == '1': |
| 1061 | write_to_bazelrc('build --config=cuda_clang') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1062 | else: |
A. Unique TensorFlower | ab39198 | 2018-07-11 04:52:49 -0700 | [diff] [blame] | 1063 | write_to_bazelrc('build --config=cuda') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1064 | |
| 1065 | |
| 1066 | def set_host_cxx_compiler(environ_cp): |
| 1067 | """Set HOST_CXX_COMPILER.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1068 | default_cxx_host_compiler = which('g++') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1069 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1070 | host_cxx_compiler = prompt_loop_or_load_from_env( |
| 1071 | environ_cp, |
| 1072 | var_name='HOST_CXX_COMPILER', |
| 1073 | var_default=default_cxx_host_compiler, |
| 1074 | ask_for_var=('Please specify which C++ compiler should be used as the ' |
| 1075 | 'host C++ compiler.'), |
| 1076 | check_success=os.path.exists, |
| 1077 | error_msg='Invalid C++ compiler path. %s cannot be found.', |
| 1078 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1079 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1080 | write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler) |
| 1081 | |
| 1082 | |
| 1083 | def set_host_c_compiler(environ_cp): |
| 1084 | """Set HOST_C_COMPILER.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1085 | default_c_host_compiler = which('gcc') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1086 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1087 | host_c_compiler = prompt_loop_or_load_from_env( |
| 1088 | environ_cp, |
| 1089 | var_name='HOST_C_COMPILER', |
| 1090 | var_default=default_c_host_compiler, |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1091 | ask_for_var=('Please specify which C compiler should be used as the host ' |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1092 | 'C compiler.'), |
| 1093 | check_success=os.path.exists, |
| 1094 | error_msg='Invalid C compiler path. %s cannot be found.', |
| 1095 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1096 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1097 | write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler) |
| 1098 | |
| 1099 | |
| 1100 | def set_computecpp_toolkit_path(environ_cp): |
| 1101 | """Set COMPUTECPP_TOOLKIT_PATH.""" |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1102 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1103 | def toolkit_exists(toolkit_path): |
| 1104 | """Check if a computecpp toolkit path is valid.""" |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1105 | if is_linux(): |
| 1106 | sycl_rt_lib_path = 'lib/libComputeCpp.so' |
| 1107 | else: |
| 1108 | sycl_rt_lib_path = '' |
| 1109 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1110 | sycl_rt_lib_path_full = os.path.join(toolkit_path, sycl_rt_lib_path) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1111 | exists = os.path.exists(sycl_rt_lib_path_full) |
| 1112 | if not exists: |
| 1113 | print('Invalid SYCL %s library path. %s cannot be found' % |
| 1114 | (_TF_OPENCL_VERSION, sycl_rt_lib_path_full)) |
| 1115 | return exists |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1116 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1117 | computecpp_toolkit_path = prompt_loop_or_load_from_env( |
| 1118 | environ_cp, |
| 1119 | var_name='COMPUTECPP_TOOLKIT_PATH', |
| 1120 | var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH, |
| 1121 | ask_for_var=( |
| 1122 | 'Please specify the location where ComputeCpp for SYCL %s is ' |
| 1123 | 'installed.' % _TF_OPENCL_VERSION), |
| 1124 | check_success=toolkit_exists, |
| 1125 | error_msg='Invalid SYCL compiler path. %s cannot be found.', |
| 1126 | suppress_default_error=True) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1127 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1128 | write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH', |
| 1129 | computecpp_toolkit_path) |
| 1130 | |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1131 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1132 | def set_trisycl_include_dir(environ_cp): |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1133 | """Set TRISYCL_INCLUDE_DIR.""" |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 1134 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1135 | ask_trisycl_include_dir = ('Please specify the location of the triSYCL ' |
| 1136 | 'include directory. (Use --config=sycl_trisycl ' |
| 1137 | 'when building with Bazel) ' |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1138 | '[Default is %s]: ') % ( |
| 1139 | _DEFAULT_TRISYCL_INCLUDE_DIR) |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 1140 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1141 | while True: |
| 1142 | trisycl_include_dir = get_from_env_or_user_or_default( |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1143 | environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir, |
| 1144 | _DEFAULT_TRISYCL_INCLUDE_DIR) |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1145 | if os.path.exists(trisycl_include_dir): |
| 1146 | break |
| 1147 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1148 | print('Invalid triSYCL include directory, %s cannot be found' % |
| 1149 | (trisycl_include_dir)) |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1150 | |
| 1151 | # Set TRISYCL_INCLUDE_DIR |
| 1152 | environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1153 | write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', trisycl_include_dir) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1154 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1155 | |
A. Unique TensorFlower | e9797fd | 2019-03-22 11:02:40 -0700 | [diff] [blame] | 1156 | def system_specific_test_config(env): |
A. Unique TensorFlower | 7bd8637 | 2019-03-21 15:19:30 -0700 | [diff] [blame] | 1157 | """Add default build and test flags required for TF tests to bazelrc.""" |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1158 | write_to_bazelrc('test --flaky_test_attempts=3') |
| 1159 | write_to_bazelrc('test --test_size_filters=small,medium') |
frreiss | 8b013a7 | 2019-11-20 11:36:27 -0800 | [diff] [blame] | 1160 | |
| 1161 | # Each instance of --test_tag_filters or --build_tag_filters overrides all |
| 1162 | # previous instances, so we need to build up a complete list and write a |
| 1163 | # single list of filters for the .bazelrc file. |
| 1164 | |
| 1165 | # Filters to use with both --test_tag_filters and --build_tag_filters |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1166 | test_and_build_filters = ['-benchmark-test', '-no_oss'] |
| 1167 | # Additional filters for --test_tag_filters beyond those in |
| 1168 | # test_and_build_filters |
| 1169 | test_only_filters = ['-oss_serial'] |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1170 | if is_windows(): |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1171 | test_and_build_filters.append('-no_windows') |
Guangda Lai | bcd701a | 2019-03-12 21:04:51 -0700 | [diff] [blame] | 1172 | if env.get('TF_NEED_CUDA', None) == '1': |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1173 | test_and_build_filters += ['-no_windows_gpu', '-no_gpu'] |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1174 | else: |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1175 | test_and_build_filters.append('-gpu') |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1176 | elif is_macos(): |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1177 | test_and_build_filters += ['-gpu', '-nomac', '-no_mac'] |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1178 | elif is_linux(): |
Guangda Lai | bcd701a | 2019-03-12 21:04:51 -0700 | [diff] [blame] | 1179 | if env.get('TF_NEED_CUDA', None) == '1': |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1180 | test_and_build_filters.append('-no_gpu') |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1181 | write_to_bazelrc('test --test_env=LD_LIBRARY_PATH') |
| 1182 | else: |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1183 | test_and_build_filters.append('-gpu') |
frreiss | 3adf417 | 2019-12-06 14:16:49 -0800 | [diff] [blame] | 1184 | |
| 1185 | # Disable tests with "v1only" tag in "v2" Bazel config, but not in "v1" config |
| 1186 | write_to_bazelrc('test:v1 --test_tag_filters=%s' % |
TensorFlower Gardener | e515908 | 2019-11-22 11:47:38 -0800 | [diff] [blame] | 1187 | ','.join(test_and_build_filters + test_only_filters)) |
frreiss | 3adf417 | 2019-12-06 14:16:49 -0800 | [diff] [blame] | 1188 | write_to_bazelrc('test:v1 --build_tag_filters=%s' % |
TensorFlower Gardener | e515908 | 2019-11-22 11:47:38 -0800 | [diff] [blame] | 1189 | ','.join(test_and_build_filters)) |
TensorFlower Gardener | 26a1de3 | 2019-12-17 07:11:48 -0800 | [diff] [blame] | 1190 | write_to_bazelrc( |
| 1191 | 'test:v2 --test_tag_filters=%s' % |
| 1192 | ','.join(test_and_build_filters + test_only_filters + ['-v1only'])) |
frreiss | 3adf417 | 2019-12-06 14:16:49 -0800 | [diff] [blame] | 1193 | write_to_bazelrc('test:v2 --build_tag_filters=%s' % |
TensorFlower Gardener | 26a1de3 | 2019-12-17 07:11:48 -0800 | [diff] [blame] | 1194 | ','.join(test_and_build_filters + ['-v1only'])) |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1195 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1196 | |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1197 | def set_system_libs_flag(environ_cp): |
| 1198 | syslibs = environ_cp.get('TF_SYSTEM_LIBS', '') |
TensorFlower Gardener | 61a8720 | 2018-10-01 12:25:39 -0700 | [diff] [blame] | 1199 | if syslibs: |
Jason Zaman | 5fc39bd | 2018-09-16 01:38:55 +0800 | [diff] [blame] | 1200 | if ',' in syslibs: |
| 1201 | syslibs = ','.join(sorted(syslibs.split(','))) |
| 1202 | else: |
| 1203 | syslibs = ','.join(sorted(syslibs.split())) |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1204 | write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs) |
| 1205 | |
Jason Zaman | 5fc39bd | 2018-09-16 01:38:55 +0800 | [diff] [blame] | 1206 | if 'PREFIX' in environ_cp: |
| 1207 | write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX']) |
| 1208 | if 'LIBDIR' in environ_cp: |
| 1209 | write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR']) |
| 1210 | if 'INCLUDEDIR' in environ_cp: |
| 1211 | write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR']) |
| 1212 | |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1213 | |
Austin Anderson | 6d7c376 | 2019-11-19 17:04:19 -0800 | [diff] [blame] | 1214 | def is_reduced_optimize_huge_functions_available(environ_cp): |
| 1215 | """Check to see if the system supports /d2ReducedOptimizeHugeFunctions. |
| 1216 | |
| 1217 | The above compiler flag is a new compiler flag introduced to the Visual Studio |
| 1218 | compiler in version 16.4 (available in Visual Studio 2019, Preview edition |
| 1219 | only, as of 2019-11-19). TensorFlow needs this flag to massively reduce |
| 1220 | compile times, but until 16.4 is officially released, we can't depend on it. |
| 1221 | |
| 1222 | See also https://groups.google.com/a/tensorflow.org/g/build/c/SsW98Eo7l3o |
| 1223 | |
| 1224 | Because it's very annoying to check this manually (to check the MSVC installed |
| 1225 | versions, you need to use the registry, and it's not clear if Bazel will be |
| 1226 | using that install version anyway), we expect enviroments who know they may |
| 1227 | use this flag to export TF_VC_VERSION=16.4 |
| 1228 | |
| 1229 | TODO(angerson, gunan): Remove this function when TensorFlow's minimum VS |
| 1230 | version is upgraded to 16.4. |
| 1231 | |
| 1232 | Arguments: |
| 1233 | environ_cp: Environment of the current execution |
| 1234 | |
| 1235 | Returns: |
| 1236 | boolean, whether or not /d2ReducedOptimizeHugeFunctions is available on this |
| 1237 | machine. |
| 1238 | """ |
| 1239 | return float(environ_cp.get('TF_VC_VERSION', '0')) >= 16.4 |
| 1240 | |
| 1241 | |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1242 | def set_windows_build_flags(environ_cp): |
| 1243 | """Set Windows specific build options.""" |
Austin Anderson | 6d7c376 | 2019-11-19 17:04:19 -0800 | [diff] [blame] | 1244 | if is_reduced_optimize_huge_functions_available(environ_cp): |
| 1245 | write_to_bazelrc( |
| 1246 | 'build --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions' |
| 1247 | ) |
| 1248 | |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1249 | if get_var( |
| 1250 | environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline', |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1251 | True, ('Would you like to override eigen strong inline for some C++ ' |
| 1252 | 'compilation to reduce the compilation time?'), |
| 1253 | 'Eigen strong inline overridden.', 'Not overriding eigen strong inline, ' |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1254 | 'some compilations could take more than 20 mins.'): |
| 1255 | # Due to a known MSVC compiler issue |
| 1256 | # https://github.com/tensorflow/tensorflow/issues/10521 |
| 1257 | # Overriding eigen strong inline speeds up the compiling of |
| 1258 | # conv_grad_ops_3d.cc and conv_ops_3d.cc by 20 minutes, |
| 1259 | # but this also hurts the performance. Let users decide what they want. |
| 1260 | write_to_bazelrc('build --define=override_eigen_strong_inline=true') |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1261 | |
A. Unique TensorFlower | 061c359 | 2017-11-13 14:21:04 -0800 | [diff] [blame] | 1262 | |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1263 | def config_info_line(name, help_text): |
| 1264 | """Helper function to print formatted help text for Bazel config options.""" |
| 1265 | print('\t--config=%-12s\t# %s' % (name, help_text)) |
| 1266 | |
| 1267 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1268 | def configure_ios(): |
| 1269 | """Configures TensorFlow for iOS builds. |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1270 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1271 | This function will only be executed if `is_macos()` is true. |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1272 | """ |
| 1273 | if not is_macos(): |
| 1274 | return |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1275 | for filepath in APPLE_BAZEL_FILES: |
| 1276 | existing_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath + '.apple') |
| 1277 | renamed_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath) |
| 1278 | symlink_force(existing_filepath, renamed_filepath) |
| 1279 | for filepath in IOS_FILES: |
| 1280 | filename = os.path.basename(filepath) |
| 1281 | new_filepath = os.path.join(_TF_WORKSPACE_ROOT, filename) |
| 1282 | symlink_force(filepath, new_filepath) |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1283 | |
| 1284 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1285 | def validate_cuda_config(environ_cp): |
| 1286 | """Run find_cuda_config.py and return cuda_toolkit_path, or None.""" |
| 1287 | |
| 1288 | def maybe_encode_env(env): |
| 1289 | """Encodes unicode in env to str on Windows python 2.x.""" |
| 1290 | if not is_windows() or sys.version_info[0] != 2: |
| 1291 | return env |
| 1292 | for k, v in env.items(): |
| 1293 | if isinstance(k, unicode): |
| 1294 | k = k.encode('ascii') |
| 1295 | if isinstance(v, unicode): |
| 1296 | v = v.encode('ascii') |
| 1297 | env[k] = v |
| 1298 | return env |
| 1299 | |
| 1300 | cuda_libraries = ['cuda', 'cudnn'] |
| 1301 | if is_linux(): |
A. Unique TensorFlower | 28fc9cc | 2019-05-01 14:17:54 -0700 | [diff] [blame] | 1302 | if int(environ_cp.get('TF_NEED_TENSORRT', False)): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1303 | cuda_libraries.append('tensorrt') |
A. Unique TensorFlower | b2e7f67 | 2019-04-30 09:20:36 -0700 | [diff] [blame] | 1304 | if environ_cp.get('TF_NCCL_VERSION', None): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1305 | cuda_libraries.append('nccl') |
| 1306 | |
| 1307 | proc = subprocess.Popen( |
| 1308 | [environ_cp['PYTHON_BIN_PATH'], 'third_party/gpus/find_cuda_config.py'] + |
| 1309 | cuda_libraries, |
| 1310 | stdout=subprocess.PIPE, |
| 1311 | env=maybe_encode_env(environ_cp)) |
| 1312 | |
| 1313 | if proc.wait(): |
| 1314 | # Errors from find_cuda_config.py were sent to stderr. |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 1315 | print('Asking for detailed CUDA configuration...\n') |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1316 | return False |
| 1317 | |
| 1318 | config = dict( |
| 1319 | tuple(line.decode('ascii').rstrip().split(': ')) for line in proc.stdout) |
| 1320 | |
| 1321 | print('Found CUDA %s in:' % config['cuda_version']) |
| 1322 | print(' %s' % config['cuda_library_dir']) |
| 1323 | print(' %s' % config['cuda_include_dir']) |
| 1324 | |
| 1325 | print('Found cuDNN %s in:' % config['cudnn_version']) |
| 1326 | print(' %s' % config['cudnn_library_dir']) |
| 1327 | print(' %s' % config['cudnn_include_dir']) |
| 1328 | |
| 1329 | if 'tensorrt_version' in config: |
| 1330 | print('Found TensorRT %s in:' % config['tensorrt_version']) |
| 1331 | print(' %s' % config['tensorrt_library_dir']) |
| 1332 | print(' %s' % config['tensorrt_include_dir']) |
| 1333 | |
| 1334 | if config.get('nccl_version', None): |
| 1335 | print('Found NCCL %s in:' % config['nccl_version']) |
| 1336 | print(' %s' % config['nccl_library_dir']) |
| 1337 | print(' %s' % config['nccl_include_dir']) |
| 1338 | |
| 1339 | print('\n') |
| 1340 | |
| 1341 | environ_cp['CUDA_TOOLKIT_PATH'] = config['cuda_toolkit_path'] |
| 1342 | return True |
| 1343 | |
| 1344 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1345 | def main(): |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1346 | global _TF_WORKSPACE_ROOT |
| 1347 | global _TF_BAZELRC |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 1348 | global _TF_CURRENT_BAZEL_VERSION |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1349 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1350 | parser = argparse.ArgumentParser() |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1351 | parser.add_argument( |
| 1352 | '--workspace', |
| 1353 | type=str, |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1354 | default=os.path.abspath(os.path.dirname(__file__)), |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1355 | help='The absolute path to your active Bazel workspace.') |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1356 | args = parser.parse_args() |
| 1357 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1358 | _TF_WORKSPACE_ROOT = args.workspace |
| 1359 | _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME) |
| 1360 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1361 | # Make a copy of os.environ to be clear when functions and getting and setting |
| 1362 | # environment variables. |
| 1363 | environ_cp = dict(os.environ) |
| 1364 | |
Mark Daoust | 44000ad | 2019-06-18 09:26:26 -0700 | [diff] [blame] | 1365 | current_bazel_version = check_bazel_version(_TF_MIN_BAZEL_VERSION, |
| 1366 | _TF_MAX_BAZEL_VERSION) |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 1367 | _TF_CURRENT_BAZEL_VERSION = convert_version_to_int(current_bazel_version) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1368 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1369 | reset_tf_configure_bazelrc() |
Yun Peng | 03e63a2 | 2018-11-07 11:18:53 +0100 | [diff] [blame] | 1370 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1371 | cleanup_makefile() |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 1372 | setup_python(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1373 | |
| 1374 | if is_windows(): |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1375 | environ_cp['TF_NEED_OPENCL_SYCL'] = '0' |
| 1376 | environ_cp['TF_NEED_COMPUTECPP'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1377 | environ_cp['TF_NEED_OPENCL'] = '0' |
| 1378 | environ_cp['TF_CUDA_CLANG'] = '0' |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 1379 | environ_cp['TF_NEED_TENSORRT'] = '0' |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1380 | # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on |
| 1381 | # Windows. |
| 1382 | environ_cp['TF_DOWNLOAD_CLANG'] = '0' |
A. Unique TensorFlower | 6e97fb3 | 2018-07-16 14:07:29 -0700 | [diff] [blame] | 1383 | environ_cp['TF_NEED_MPI'] = '0' |
| 1384 | environ_cp['TF_SET_ANDROID_WORKSPACE'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1385 | |
| 1386 | if is_macos(): |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 1387 | environ_cp['TF_NEED_TENSORRT'] = '0' |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1388 | else: |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1389 | environ_cp['TF_CONFIGURE_IOS'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1390 | |
Sanjoy Das | f3fab7c | 2019-06-20 11:19:00 -0700 | [diff] [blame] | 1391 | xla_enabled_by_default = is_linux() or is_macos() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1392 | set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support', |
Grzegorz Pawelczak | ec82efd | 2018-10-09 15:03:46 +0100 | [diff] [blame] | 1393 | xla_enabled_by_default, 'xla') |
Gunhan Gulsoy | 3da0dff | 2018-09-26 11:55:50 -0700 | [diff] [blame] | 1394 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1395 | set_action_env_var( |
| 1396 | environ_cp, |
| 1397 | 'TF_NEED_OPENCL_SYCL', |
| 1398 | 'OpenCL SYCL', |
| 1399 | False, |
| 1400 | bazel_config_name='sycl') |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1401 | if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1402 | set_host_cxx_compiler(environ_cp) |
| 1403 | set_host_c_compiler(environ_cp) |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1404 | set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True) |
| 1405 | if environ_cp.get('TF_NEED_COMPUTECPP') == '1': |
| 1406 | set_computecpp_toolkit_path(environ_cp) |
| 1407 | else: |
| 1408 | set_trisycl_include_dir(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1409 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1410 | set_action_env_var( |
| 1411 | environ_cp, 'TF_NEED_ROCM', 'ROCm', False, bazel_config_name='rocm') |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1412 | if (environ_cp.get('TF_NEED_ROCM') == '1' and |
TensorFlower Gardener | 62e6016 | 2018-09-27 10:22:55 -0700 | [diff] [blame] | 1413 | 'LD_LIBRARY_PATH' in environ_cp and |
| 1414 | environ_cp.get('LD_LIBRARY_PATH') != '1'): |
| 1415 | write_action_env_to_bazelrc('LD_LIBRARY_PATH', |
| 1416 | environ_cp.get('LD_LIBRARY_PATH')) |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1417 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1418 | environ_cp['TF_NEED_CUDA'] = str( |
| 1419 | int(get_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False))) |
A. Unique TensorFlower | 24cbb2a | 2017-09-08 07:45:44 -0700 | [diff] [blame] | 1420 | if (environ_cp.get('TF_NEED_CUDA') == '1' and |
| 1421 | 'TF_CUDA_CONFIG_REPO' not in environ_cp): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1422 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1423 | set_action_env_var( |
| 1424 | environ_cp, |
| 1425 | 'TF_NEED_TENSORRT', |
| 1426 | 'TensorRT', |
| 1427 | False, |
| 1428 | bazel_config_name='tensorrt') |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1429 | |
| 1430 | environ_save = dict(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1431 | for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1432 | |
| 1433 | if validate_cuda_config(environ_cp): |
| 1434 | cuda_env_names = [ |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1435 | 'TF_CUDA_VERSION', |
| 1436 | 'TF_CUBLAS_VERSION', |
| 1437 | 'TF_CUDNN_VERSION', |
| 1438 | 'TF_TENSORRT_VERSION', |
| 1439 | 'TF_NCCL_VERSION', |
| 1440 | 'TF_CUDA_PATHS', |
A. Unique TensorFlower | b2e7f67 | 2019-04-30 09:20:36 -0700 | [diff] [blame] | 1441 | # Items below are for backwards compatibility when not using |
| 1442 | # TF_CUDA_PATHS. |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1443 | 'CUDA_TOOLKIT_PATH', |
| 1444 | 'CUDNN_INSTALL_PATH', |
| 1445 | 'NCCL_INSTALL_PATH', |
| 1446 | 'NCCL_HDR_PATH', |
| 1447 | 'TENSORRT_INSTALL_PATH' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1448 | ] |
A. Unique TensorFlower | b2e7f67 | 2019-04-30 09:20:36 -0700 | [diff] [blame] | 1449 | # Note: set_action_env_var above already writes to bazelrc. |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1450 | for name in cuda_env_names: |
| 1451 | if name in environ_cp: |
| 1452 | write_action_env_to_bazelrc(name, environ_cp[name]) |
| 1453 | break |
| 1454 | |
| 1455 | # Restore settings changed below if CUDA config could not be validated. |
| 1456 | environ_cp = dict(environ_save) |
| 1457 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1458 | set_tf_cuda_version(environ_cp) |
| 1459 | set_tf_cudnn_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1460 | if is_linux(): |
| 1461 | set_tf_tensorrt_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1462 | set_tf_nccl_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1463 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1464 | set_tf_cuda_paths(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1465 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1466 | else: |
| 1467 | raise UserInputError( |
| 1468 | 'Invalid CUDA setting were provided %d ' |
| 1469 | 'times in a row. Assuming to be a scripting mistake.' % |
| 1470 | _DEFAULT_PROMPT_ASK_ATTEMPTS) |
| 1471 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1472 | set_tf_cuda_compute_capabilities(environ_cp) |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 1473 | if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get( |
| 1474 | 'LD_LIBRARY_PATH') != '1': |
| 1475 | write_action_env_to_bazelrc('LD_LIBRARY_PATH', |
| 1476 | environ_cp.get('LD_LIBRARY_PATH')) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1477 | |
| 1478 | set_tf_cuda_clang(environ_cp) |
| 1479 | if environ_cp.get('TF_CUDA_CLANG') == '1': |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1480 | # Ask whether we should download the clang toolchain. |
| 1481 | set_tf_download_clang(environ_cp) |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 1482 | if environ_cp.get('TF_DOWNLOAD_CLANG') != '1': |
| 1483 | # Set up which clang we should use as the cuda / host compiler. |
| 1484 | set_clang_cuda_compiler_path(environ_cp) |
Ilya Biryukov | 1c3d02e | 2018-09-04 03:09:52 -0700 | [diff] [blame] | 1485 | else: |
| 1486 | # Use downloaded LLD for linking. |
| 1487 | write_to_bazelrc('build:cuda_clang --config=download_clang_use_lld') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1488 | else: |
| 1489 | # Set up which gcc nvcc should use as the host compiler |
| 1490 | # No need to set this on Windows |
| 1491 | if not is_windows(): |
| 1492 | set_gcc_host_compiler_path(environ_cp) |
| 1493 | set_other_cuda_vars(environ_cp) |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1494 | else: |
| 1495 | # CUDA not required. Ask whether we should download the clang toolchain and |
| 1496 | # use it for the CPU build. |
| 1497 | set_tf_download_clang(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1498 | |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1499 | # SYCL / ROCm / CUDA are mutually exclusive. |
| 1500 | # At most 1 GPU platform can be configured. |
| 1501 | gpu_platform_count = 0 |
| 1502 | if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': |
| 1503 | gpu_platform_count += 1 |
| 1504 | if environ_cp.get('TF_NEED_ROCM') == '1': |
| 1505 | gpu_platform_count += 1 |
| 1506 | if environ_cp.get('TF_NEED_CUDA') == '1': |
| 1507 | gpu_platform_count += 1 |
| 1508 | if gpu_platform_count >= 2: |
| 1509 | raise UserInputError('SYCL / CUDA / ROCm are mututally exclusive. ' |
| 1510 | 'At most 1 GPU platform can be configured.') |
| 1511 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1512 | set_cc_opt_flags(environ_cp) |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1513 | set_system_libs_flag(environ_cp) |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1514 | if is_windows(): |
| 1515 | set_windows_build_flags(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1516 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1517 | if get_var(environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', False, |
| 1518 | ('Would you like to interactively configure ./WORKSPACE for ' |
| 1519 | 'Android builds?'), 'Searching for NDK and SDK installations.', |
| 1520 | 'Not configuring the WORKSPACE for Android builds.'): |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 1521 | create_android_ndk_rule(environ_cp) |
| 1522 | create_android_sdk_rule(environ_cp) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1523 | |
A. Unique TensorFlower | e9797fd | 2019-03-22 11:02:40 -0700 | [diff] [blame] | 1524 | system_specific_test_config(os.environ) |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1525 | |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 1526 | set_action_env_var(environ_cp, 'TF_CONFIGURE_IOS', 'iOS', False) |
| 1527 | if environ_cp.get('TF_CONFIGURE_IOS') == '1': |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1528 | configure_ios() |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1529 | |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1530 | print('Preconfigured Bazel build configs. You can use any of the below by ' |
| 1531 | 'adding "--config=<>" to your build command. See .bazelrc for more ' |
| 1532 | 'details.') |
| 1533 | config_info_line('mkl', 'Build with MKL support.') |
| 1534 | config_info_line('monolithic', 'Config for mostly static monolithic build.') |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1535 | config_info_line('ngraph', 'Build with Intel nGraph support.') |
A. Unique TensorFlower | a6bf9c8 | 2019-02-26 10:08:35 -0800 | [diff] [blame] | 1536 | config_info_line('numa', 'Build with NUMA support.') |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 1537 | config_info_line( |
| 1538 | 'dynamic_kernels', |
| 1539 | '(Experimental) Build kernels into separate shared objects.') |
Anna R | aedf674 | 2019-06-12 11:30:51 -0700 | [diff] [blame] | 1540 | config_info_line('v2', 'Build TensorFlow 2.x instead of 1.x.') |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1541 | |
| 1542 | print('Preconfigured Bazel build configs to DISABLE default on features:') |
| 1543 | config_info_line('noaws', 'Disable AWS S3 filesystem support.') |
| 1544 | config_info_line('nogcp', 'Disable GCP support.') |
| 1545 | config_info_line('nohdfs', 'Disable HDFS support.') |
Gunhan Gulsoy | eea8168 | 2018-11-26 16:51:23 -0800 | [diff] [blame] | 1546 | config_info_line('nonccl', 'Disable NVIDIA NCCL support.') |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1547 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1548 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1549 | if __name__ == '__main__': |
| 1550 | main() |