A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | # ============================================================================== |
| 15 | """configure script to get build parameters from user.""" |
| 16 | |
| 17 | from __future__ import absolute_import |
| 18 | from __future__ import division |
| 19 | from __future__ import print_function |
| 20 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 21 | import argparse |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 22 | import errno |
| 23 | import os |
| 24 | import platform |
| 25 | import re |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 26 | import subprocess |
| 27 | import sys |
| 28 | |
Andrew Selle | c9885ea | 2017-11-06 09:37:03 -0800 | [diff] [blame] | 29 | # pylint: disable=g-import-not-at-top |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 30 | try: |
| 31 | from shutil import which |
| 32 | except ImportError: |
| 33 | from distutils.spawn import find_executable as which |
Andrew Selle | c9885ea | 2017-11-06 09:37:03 -0800 | [diff] [blame] | 34 | # pylint: enable=g-import-not-at-top |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 35 | |
A. Unique TensorFlower | 7ed3830 | 2019-11-13 11:14:29 -0800 | [diff] [blame] | 36 | _DEFAULT_CUDA_VERSION = '10.1' |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 37 | _DEFAULT_CUDNN_VERSION = '7' |
Guangda Lai | faa93ac | 2019-10-23 15:07:12 -0700 | [diff] [blame] | 38 | _DEFAULT_TENSORRT_VERSION = '6' |
Smit Hinsu | fe7d1d9 | 2018-07-14 13:16:58 -0700 | [diff] [blame] | 39 | _DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 40 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 41 | _TF_OPENCL_VERSION = '1.2' |
| 42 | _DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp' |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 43 | _DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include' |
A. Unique TensorFlower | 82820ef | 2018-11-12 13:22:13 -0800 | [diff] [blame] | 44 | _SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15, 16, 17, 18] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 45 | |
| 46 | _DEFAULT_PROMPT_ASK_ATTEMPTS = 10 |
| 47 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 48 | _TF_BAZELRC_FILENAME = '.tf_configure.bazelrc' |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 49 | _TF_WORKSPACE_ROOT = '' |
| 50 | _TF_BAZELRC = '' |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 51 | _TF_CURRENT_BAZEL_VERSION = None |
A. Unique TensorFlower | 72d0fac | 2019-11-06 06:24:43 -0800 | [diff] [blame] | 52 | _TF_MIN_BAZEL_VERSION = '0.27.1' |
A. Unique TensorFlower | a5f9bcd | 2019-11-12 07:08:54 -0800 | [diff] [blame] | 53 | _TF_MAX_BAZEL_VERSION = '1.1.0' |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 54 | |
Jason Furmanek | 7c23415 | 2018-09-26 04:44:12 +0000 | [diff] [blame] | 55 | NCCL_LIB_PATHS = [ |
TensorFlower Gardener | ea5c529 | 2018-10-01 17:28:26 -0700 | [diff] [blame] | 56 | 'lib64/', 'lib/powerpc64le-linux-gnu/', 'lib/x86_64-linux-gnu/', '' |
Jason Furmanek | 7c23415 | 2018-09-26 04:44:12 +0000 | [diff] [blame] | 57 | ] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 58 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 59 | # List of files to configure when building Bazel on Apple platforms. |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 60 | APPLE_BAZEL_FILES = [ |
A. Unique TensorFlower | 6a059c0 | 2019-04-05 14:28:27 -0700 | [diff] [blame] | 61 | 'tensorflow/lite/experimental/ios/BUILD', |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 62 | 'tensorflow/lite/experimental/objc/BUILD', |
YoungSeok Yoon | 7089aaa | 2019-11-11 21:36:17 -0800 | [diff] [blame] | 63 | 'tensorflow/lite/experimental/swift/BUILD', |
| 64 | 'tensorflow/lite/tools/benchmark/experimental/ios/BUILD' |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 65 | ] |
| 66 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 67 | # List of files to move when building for iOS. |
| 68 | IOS_FILES = [ |
| 69 | 'tensorflow/lite/experimental/objc/TensorFlowLiteObjC.podspec', |
| 70 | 'tensorflow/lite/experimental/swift/TensorFlowLiteSwift.podspec', |
| 71 | ] |
| 72 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 73 | |
| 74 | class UserInputError(Exception): |
| 75 | pass |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 76 | |
| 77 | |
| 78 | def is_windows(): |
| 79 | return platform.system() == 'Windows' |
| 80 | |
| 81 | |
| 82 | def is_linux(): |
| 83 | return platform.system() == 'Linux' |
| 84 | |
| 85 | |
| 86 | def is_macos(): |
| 87 | return platform.system() == 'Darwin' |
| 88 | |
| 89 | |
| 90 | def is_ppc64le(): |
| 91 | return platform.machine() == 'ppc64le' |
| 92 | |
| 93 | |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 94 | def is_cygwin(): |
| 95 | return platform.system().startswith('CYGWIN_NT') |
| 96 | |
| 97 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 98 | def get_input(question): |
| 99 | try: |
| 100 | try: |
| 101 | answer = raw_input(question) |
| 102 | except NameError: |
| 103 | answer = input(question) # pylint: disable=bad-builtin |
| 104 | except EOFError: |
| 105 | answer = '' |
| 106 | return answer |
| 107 | |
| 108 | |
| 109 | def symlink_force(target, link_name): |
| 110 | """Force symlink, equivalent of 'ln -sf'. |
| 111 | |
| 112 | Args: |
| 113 | target: items to link to. |
| 114 | link_name: name of the link. |
| 115 | """ |
| 116 | try: |
| 117 | os.symlink(target, link_name) |
| 118 | except OSError as e: |
| 119 | if e.errno == errno.EEXIST: |
| 120 | os.remove(link_name) |
| 121 | os.symlink(target, link_name) |
| 122 | else: |
| 123 | raise e |
| 124 | |
| 125 | |
| 126 | def sed_in_place(filename, old, new): |
| 127 | """Replace old string with new string in file. |
| 128 | |
| 129 | Args: |
| 130 | filename: string for filename. |
| 131 | old: string to replace. |
| 132 | new: new string to replace to. |
| 133 | """ |
| 134 | with open(filename, 'r') as f: |
| 135 | filedata = f.read() |
| 136 | newdata = filedata.replace(old, new) |
| 137 | with open(filename, 'w') as f: |
| 138 | f.write(newdata) |
| 139 | |
| 140 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 141 | def write_to_bazelrc(line): |
| 142 | with open(_TF_BAZELRC, 'a') as f: |
| 143 | f.write(line + '\n') |
| 144 | |
| 145 | |
| 146 | def write_action_env_to_bazelrc(var_name, var): |
| 147 | write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var))) |
| 148 | |
| 149 | |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 150 | def run_shell(cmd, allow_non_zero=False): |
| 151 | if allow_non_zero: |
| 152 | try: |
| 153 | output = subprocess.check_output(cmd) |
| 154 | except subprocess.CalledProcessError as e: |
| 155 | output = e.output |
| 156 | else: |
| 157 | output = subprocess.check_output(cmd) |
| 158 | return output.decode('UTF-8').strip() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 159 | |
| 160 | |
| 161 | def cygpath(path): |
| 162 | """Convert path from posix to windows.""" |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 163 | return os.path.abspath(path).replace('\\', '/') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 164 | |
| 165 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 166 | def get_python_path(environ_cp, python_bin_path): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 167 | """Get the python site package paths.""" |
| 168 | python_paths = [] |
| 169 | if environ_cp.get('PYTHONPATH'): |
| 170 | python_paths = environ_cp.get('PYTHONPATH').split(':') |
| 171 | try: |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 172 | library_paths = run_shell([ |
| 173 | python_bin_path, '-c', |
| 174 | 'import site; print("\\n".join(site.getsitepackages()))' |
| 175 | ]).split('\n') |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 176 | except subprocess.CalledProcessError: |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 177 | library_paths = [ |
| 178 | run_shell([ |
| 179 | python_bin_path, '-c', |
| 180 | 'from distutils.sysconfig import get_python_lib;' |
| 181 | 'print(get_python_lib())' |
| 182 | ]) |
| 183 | ] |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 184 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 185 | all_paths = set(python_paths + library_paths) |
| 186 | |
| 187 | paths = [] |
| 188 | for path in all_paths: |
| 189 | if os.path.isdir(path): |
| 190 | paths.append(path) |
| 191 | return paths |
| 192 | |
| 193 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 194 | def get_python_major_version(python_bin_path): |
| 195 | """Get the python major version.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 196 | return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])']) |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 197 | |
| 198 | |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 199 | def setup_python(environ_cp): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 200 | """Setup python related env variables.""" |
| 201 | # Get PYTHON_BIN_PATH, default is the current running python. |
| 202 | default_python_bin_path = sys.executable |
| 203 | ask_python_bin_path = ('Please specify the location of python. [Default is ' |
| 204 | '%s]: ') % default_python_bin_path |
| 205 | while True: |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 206 | python_bin_path = get_from_env_or_user_or_default(environ_cp, |
| 207 | 'PYTHON_BIN_PATH', |
| 208 | ask_python_bin_path, |
| 209 | default_python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 210 | # Check if the path is valid |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 211 | if os.path.isfile(python_bin_path) and os.access(python_bin_path, os.X_OK): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 212 | break |
| 213 | elif not os.path.exists(python_bin_path): |
| 214 | print('Invalid python path: %s cannot be found.' % python_bin_path) |
| 215 | else: |
| 216 | print('%s is not executable. Is it the python binary?' % python_bin_path) |
| 217 | environ_cp['PYTHON_BIN_PATH'] = '' |
| 218 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 219 | # Convert python path to Windows style before checking lib and version |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 220 | if is_windows() or is_cygwin(): |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 221 | python_bin_path = cygpath(python_bin_path) |
| 222 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 223 | # Get PYTHON_LIB_PATH |
| 224 | python_lib_path = environ_cp.get('PYTHON_LIB_PATH') |
| 225 | if not python_lib_path: |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 226 | python_lib_paths = get_python_path(environ_cp, python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 227 | if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1': |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 228 | python_lib_path = python_lib_paths[0] |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 229 | else: |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 230 | print('Found possible Python library paths:\n %s' % |
| 231 | '\n '.join(python_lib_paths)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 232 | default_python_lib_path = python_lib_paths[0] |
| 233 | python_lib_path = get_input( |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 234 | 'Please input the desired Python library path to use. ' |
| 235 | 'Default is [%s]\n' % python_lib_paths[0]) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 236 | if not python_lib_path: |
| 237 | python_lib_path = default_python_lib_path |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 238 | environ_cp['PYTHON_LIB_PATH'] = python_lib_path |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 239 | |
A. Unique TensorFlower | cfccbdb | 2019-06-24 11:01:58 -0700 | [diff] [blame] | 240 | python_major_version = get_python_major_version(python_bin_path) |
| 241 | if python_major_version == '2': |
| 242 | write_to_bazelrc('build --host_force_python=PY2') |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 243 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 244 | # Convert python path to Windows style before writing into bazel.rc |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 245 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 246 | python_lib_path = cygpath(python_lib_path) |
| 247 | |
| 248 | # Set-up env variables used by python_configure.bzl |
| 249 | write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path) |
| 250 | write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path) |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 251 | write_to_bazelrc('build --python_path=\"%s"' % python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 252 | environ_cp['PYTHON_BIN_PATH'] = python_bin_path |
| 253 | |
William D. Irons | dcc76a5 | 2018-11-20 10:35:18 -0600 | [diff] [blame] | 254 | # If choosen python_lib_path is from a path specified in the PYTHONPATH |
| 255 | # variable, need to tell bazel to include PYTHONPATH |
| 256 | if environ_cp.get('PYTHONPATH'): |
| 257 | python_paths = environ_cp.get('PYTHONPATH').split(':') |
| 258 | if python_lib_path in python_paths: |
TensorFlower Gardener | 968cd18 | 2018-11-28 11:33:16 -0800 | [diff] [blame] | 259 | write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH')) |
William D. Irons | dcc76a5 | 2018-11-20 10:35:18 -0600 | [diff] [blame] | 260 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 261 | # Write tools/python_bin_path.sh |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 262 | with open( |
| 263 | os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'), |
| 264 | 'w') as f: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 265 | f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path) |
| 266 | |
| 267 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 268 | def reset_tf_configure_bazelrc(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 269 | """Reset file that contains customized config settings.""" |
| 270 | open(_TF_BAZELRC, 'w').close() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 271 | |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 272 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 273 | def cleanup_makefile(): |
| 274 | """Delete any leftover BUILD files from the Makefile build. |
| 275 | |
| 276 | These files could interfere with Bazel parsing. |
| 277 | """ |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 278 | makefile_download_dir = os.path.join(_TF_WORKSPACE_ROOT, 'tensorflow', |
| 279 | 'contrib', 'makefile', 'downloads') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 280 | if os.path.isdir(makefile_download_dir): |
| 281 | for root, _, filenames in os.walk(makefile_download_dir): |
| 282 | for f in filenames: |
| 283 | if f.endswith('BUILD'): |
| 284 | os.remove(os.path.join(root, f)) |
| 285 | |
| 286 | |
| 287 | def get_var(environ_cp, |
| 288 | var_name, |
| 289 | query_item, |
| 290 | enabled_by_default, |
| 291 | question=None, |
| 292 | yes_reply=None, |
| 293 | no_reply=None): |
| 294 | """Get boolean input from user. |
| 295 | |
| 296 | If var_name is not set in env, ask user to enable query_item or not. If the |
| 297 | response is empty, use the default. |
| 298 | |
| 299 | Args: |
| 300 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 301 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
| 302 | query_item: string for feature related to the variable, e.g. "CUDA for |
| 303 | Nvidia GPUs". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 304 | enabled_by_default: boolean for default behavior. |
| 305 | question: optional string for how to ask for user input. |
Michael Case | d90054e | 2018-02-07 14:36:00 -0800 | [diff] [blame] | 306 | yes_reply: optional string for reply when feature is enabled. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 307 | no_reply: optional string for reply when feature is disabled. |
| 308 | |
| 309 | Returns: |
| 310 | boolean value of the variable. |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 311 | |
| 312 | Raises: |
| 313 | UserInputError: if an environment variable is set, but it cannot be |
| 314 | interpreted as a boolean indicator, assume that the user has made a |
| 315 | scripting error, and will continue to provide invalid input. |
| 316 | Raise the error to avoid infinitely looping. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 317 | """ |
| 318 | if not question: |
| 319 | question = 'Do you wish to build TensorFlow with %s support?' % query_item |
| 320 | if not yes_reply: |
| 321 | yes_reply = '%s support will be enabled for TensorFlow.' % query_item |
| 322 | if not no_reply: |
| 323 | no_reply = 'No %s' % yes_reply |
| 324 | |
| 325 | yes_reply += '\n' |
| 326 | no_reply += '\n' |
| 327 | |
| 328 | if enabled_by_default: |
| 329 | question += ' [Y/n]: ' |
| 330 | else: |
| 331 | question += ' [y/N]: ' |
| 332 | |
| 333 | var = environ_cp.get(var_name) |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 334 | if var is not None: |
| 335 | var_content = var.strip().lower() |
| 336 | true_strings = ('1', 't', 'true', 'y', 'yes') |
| 337 | false_strings = ('0', 'f', 'false', 'n', 'no') |
| 338 | if var_content in true_strings: |
| 339 | var = True |
| 340 | elif var_content in false_strings: |
| 341 | var = False |
| 342 | else: |
| 343 | raise UserInputError( |
| 344 | 'Environment variable %s must be set as a boolean indicator.\n' |
| 345 | 'The following are accepted as TRUE : %s.\n' |
| 346 | 'The following are accepted as FALSE: %s.\n' |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 347 | 'Current value is %s.' % |
| 348 | (var_name, ', '.join(true_strings), ', '.join(false_strings), var)) |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 349 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 350 | while var is None: |
| 351 | user_input_origin = get_input(question) |
| 352 | user_input = user_input_origin.strip().lower() |
| 353 | if user_input == 'y': |
| 354 | print(yes_reply) |
| 355 | var = True |
| 356 | elif user_input == 'n': |
| 357 | print(no_reply) |
| 358 | var = False |
| 359 | elif not user_input: |
| 360 | if enabled_by_default: |
| 361 | print(yes_reply) |
| 362 | var = True |
| 363 | else: |
| 364 | print(no_reply) |
| 365 | var = False |
| 366 | else: |
| 367 | print('Invalid selection: %s' % user_input_origin) |
| 368 | return var |
| 369 | |
| 370 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 371 | def set_build_var(environ_cp, |
| 372 | var_name, |
| 373 | query_item, |
| 374 | option_name, |
| 375 | enabled_by_default, |
| 376 | bazel_config_name=None): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 377 | """Set if query_item will be enabled for the build. |
| 378 | |
| 379 | Ask user if query_item will be enabled. Default is used if no input is given. |
| 380 | Set subprocess environment variable and write to .bazelrc if enabled. |
| 381 | |
| 382 | Args: |
| 383 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 384 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
| 385 | query_item: string for feature related to the variable, e.g. "CUDA for |
| 386 | Nvidia GPUs". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 387 | option_name: string for option to define in .bazelrc. |
| 388 | enabled_by_default: boolean for default behavior. |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 389 | bazel_config_name: Name for Bazel --config argument to enable build feature. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 390 | """ |
| 391 | |
| 392 | var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default))) |
| 393 | environ_cp[var_name] = var |
| 394 | if var == '1': |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 395 | write_to_bazelrc('build:%s --define %s=true' % |
| 396 | (bazel_config_name, option_name)) |
Yifei Feng | ec451f5 | 2018-10-05 12:53:50 -0700 | [diff] [blame] | 397 | write_to_bazelrc('build --config=%s' % bazel_config_name) |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 398 | elif bazel_config_name is not None: |
| 399 | # TODO(mikecase): Migrate all users of configure.py to use --config Bazel |
| 400 | # options and not to set build configs through environment variables. |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 401 | write_to_bazelrc('build:%s --define %s=true' % |
| 402 | (bazel_config_name, option_name)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 403 | |
| 404 | |
| 405 | def set_action_env_var(environ_cp, |
| 406 | var_name, |
| 407 | query_item, |
| 408 | enabled_by_default, |
| 409 | question=None, |
| 410 | yes_reply=None, |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 411 | no_reply=None, |
| 412 | bazel_config_name=None): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 413 | """Set boolean action_env variable. |
| 414 | |
| 415 | Ask user if query_item will be enabled. Default is used if no input is given. |
| 416 | Set environment variable and write to .bazelrc. |
| 417 | |
| 418 | Args: |
| 419 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 420 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
| 421 | query_item: string for feature related to the variable, e.g. "CUDA for |
| 422 | Nvidia GPUs". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 423 | enabled_by_default: boolean for default behavior. |
| 424 | question: optional string for how to ask for user input. |
Michael Case | d90054e | 2018-02-07 14:36:00 -0800 | [diff] [blame] | 425 | yes_reply: optional string for reply when feature is enabled. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 426 | no_reply: optional string for reply when feature is disabled. |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 427 | bazel_config_name: adding config to .bazelrc instead of action_env. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 428 | """ |
| 429 | var = int( |
| 430 | get_var(environ_cp, var_name, query_item, enabled_by_default, question, |
| 431 | yes_reply, no_reply)) |
| 432 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 433 | if not bazel_config_name: |
| 434 | write_action_env_to_bazelrc(var_name, var) |
| 435 | elif var: |
| 436 | write_to_bazelrc('build --config=%s' % bazel_config_name) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 437 | environ_cp[var_name] = str(var) |
| 438 | |
| 439 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 440 | def convert_version_to_int(version): |
| 441 | """Convert a version number to a integer that can be used to compare. |
| 442 | |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 443 | Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The |
| 444 | 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored. |
| 445 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 446 | Args: |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 447 | version: a version to be converted |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 448 | |
| 449 | Returns: |
| 450 | An integer if converted successfully, otherwise return None. |
| 451 | """ |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 452 | version = version.split('-')[0] |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 453 | version_segments = version.split('.') |
Austin Anderson | 87ea41d | 2019-04-04 10:03:50 -0700 | [diff] [blame] | 454 | # Treat "0.24" as "0.24.0" |
| 455 | if len(version_segments) == 2: |
| 456 | version_segments.append('0') |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 457 | for seg in version_segments: |
| 458 | if not seg.isdigit(): |
| 459 | return None |
| 460 | |
| 461 | version_str = ''.join(['%03d' % int(seg) for seg in version_segments]) |
| 462 | return int(version_str) |
| 463 | |
| 464 | |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 465 | def check_bazel_version(min_version, max_version): |
| 466 | """Check installed bazel version is between min_version and max_version. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 467 | |
| 468 | Args: |
Mihai Maruseac | 4db860f | 2019-04-19 08:52:10 -0700 | [diff] [blame] | 469 | min_version: string for minimum bazel version (must exist!). |
| 470 | max_version: string for maximum bazel version (must exist!). |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 471 | |
| 472 | Returns: |
| 473 | The bazel version detected. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 474 | """ |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 475 | if which('bazel') is None: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 476 | print('Cannot find bazel. Please install bazel.') |
| 477 | sys.exit(0) |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 478 | curr_version = run_shell( |
| 479 | ['bazel', '--batch', '--bazelrc=/dev/null', 'version']) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 480 | |
| 481 | for line in curr_version.split('\n'): |
| 482 | if 'Build label: ' in line: |
| 483 | curr_version = line.split('Build label: ')[1] |
| 484 | break |
| 485 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 486 | min_version_int = convert_version_to_int(min_version) |
| 487 | curr_version_int = convert_version_to_int(curr_version) |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 488 | max_version_int = convert_version_to_int(max_version) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 489 | |
| 490 | # Check if current bazel version can be detected properly. |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 491 | if not curr_version_int: |
| 492 | print('WARNING: current bazel installation is not a release version.') |
| 493 | print('Make sure you are running at least bazel %s' % min_version) |
| 494 | return curr_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 495 | |
Michael Case | d94271a | 2017-08-22 17:26:52 -0700 | [diff] [blame] | 496 | print('You have bazel %s installed.' % curr_version) |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 497 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 498 | if curr_version_int < min_version_int: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 499 | print('Please upgrade your bazel installation to version %s or higher to ' |
| 500 | 'build TensorFlow!' % min_version) |
Jason Zaman | b41761c | 2018-10-14 11:28:53 +0800 | [diff] [blame] | 501 | sys.exit(1) |
TensorFlower Gardener | 78c246b | 2018-12-13 12:37:42 -0800 | [diff] [blame] | 502 | if (curr_version_int > max_version_int and |
| 503 | 'TF_IGNORE_MAX_BAZEL_VERSION' not in os.environ): |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 504 | print('Please downgrade your bazel installation to version %s or lower to ' |
Mihai Maruseac | e0963c4 | 2018-12-20 14:27:40 -0800 | [diff] [blame] | 505 | 'build TensorFlow! To downgrade: download the installer for the old ' |
| 506 | 'version (from https://github.com/bazelbuild/bazel/releases) then ' |
| 507 | 'run the installer.' % max_version) |
Jason Zaman | b41761c | 2018-10-14 11:28:53 +0800 | [diff] [blame] | 508 | sys.exit(1) |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 509 | return curr_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 510 | |
| 511 | |
| 512 | def set_cc_opt_flags(environ_cp): |
| 513 | """Set up architecture-dependent optimization flags. |
| 514 | |
| 515 | Also append CC optimization flags to bazel.rc.. |
| 516 | |
| 517 | Args: |
| 518 | environ_cp: copy of the os.environ. |
| 519 | """ |
| 520 | if is_ppc64le(): |
| 521 | # gcc on ppc64le does not support -march, use mcpu instead |
| 522 | default_cc_opt_flags = '-mcpu=native' |
A. Unique TensorFlower | 1bba94a | 2018-04-04 15:45:20 -0700 | [diff] [blame] | 523 | elif is_windows(): |
| 524 | default_cc_opt_flags = '/arch:AVX' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 525 | else: |
Justin Lebar | 9ef04f5 | 2018-10-10 18:52:45 -0700 | [diff] [blame] | 526 | default_cc_opt_flags = '-march=native -Wno-sign-compare' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 527 | question = ('Please specify optimization flags to use during compilation when' |
| 528 | ' bazel option "--config=opt" is specified [Default is %s]: ' |
| 529 | ) % default_cc_opt_flags |
| 530 | cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS', |
| 531 | question, default_cc_opt_flags) |
| 532 | for opt in cc_opt_flags.split(): |
Michael Case | 0017742 | 2017-11-10 13:14:03 -0800 | [diff] [blame] | 533 | write_to_bazelrc('build:opt --copt=%s' % opt) |
| 534 | # It should be safe on the same build host. |
A. Unique TensorFlower | 1bba94a | 2018-04-04 15:45:20 -0700 | [diff] [blame] | 535 | if not is_ppc64le() and not is_windows(): |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 536 | write_to_bazelrc('build:opt --host_copt=-march=native') |
Michael Case | bb3355d | 2017-11-09 08:46:31 -0800 | [diff] [blame] | 537 | write_to_bazelrc('build:opt --define with_default_optimizations=true') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 538 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 539 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 540 | def set_tf_cuda_clang(environ_cp): |
| 541 | """set TF_CUDA_CLANG action_env. |
| 542 | |
| 543 | Args: |
| 544 | environ_cp: copy of the os.environ. |
| 545 | """ |
| 546 | question = 'Do you want to use clang as CUDA compiler?' |
| 547 | yes_reply = 'Clang will be used as CUDA compiler.' |
| 548 | no_reply = 'nvcc will be used as CUDA compiler.' |
| 549 | set_action_env_var( |
| 550 | environ_cp, |
| 551 | 'TF_CUDA_CLANG', |
| 552 | None, |
| 553 | False, |
| 554 | question=question, |
| 555 | yes_reply=yes_reply, |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 556 | no_reply=no_reply, |
| 557 | bazel_config_name='cuda_clang') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 558 | |
| 559 | |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 560 | def set_tf_download_clang(environ_cp): |
| 561 | """Set TF_DOWNLOAD_CLANG action_env.""" |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 562 | question = 'Do you wish to download a fresh release of clang? (Experimental)' |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 563 | yes_reply = 'Clang will be downloaded and used to compile tensorflow.' |
| 564 | no_reply = 'Clang will not be downloaded.' |
| 565 | set_action_env_var( |
| 566 | environ_cp, |
| 567 | 'TF_DOWNLOAD_CLANG', |
| 568 | None, |
| 569 | False, |
| 570 | question=question, |
| 571 | yes_reply=yes_reply, |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 572 | no_reply=no_reply, |
| 573 | bazel_config_name='download_clang') |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 574 | |
| 575 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 576 | def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var, |
| 577 | var_default): |
| 578 | """Get var_name either from env, or user or default. |
| 579 | |
| 580 | If var_name has been set as environment variable, use the preset value, else |
| 581 | ask for user input. If no input is provided, the default is used. |
| 582 | |
| 583 | Args: |
| 584 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 585 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 586 | ask_for_var: string for how to ask for user input. |
| 587 | var_default: default value string. |
| 588 | |
| 589 | Returns: |
| 590 | string value for var_name |
| 591 | """ |
| 592 | var = environ_cp.get(var_name) |
| 593 | if not var: |
| 594 | var = get_input(ask_for_var) |
Michael Case | d94271a | 2017-08-22 17:26:52 -0700 | [diff] [blame] | 595 | print('\n') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 596 | if not var: |
| 597 | var = var_default |
| 598 | return var |
| 599 | |
| 600 | |
| 601 | def set_clang_cuda_compiler_path(environ_cp): |
| 602 | """Set CLANG_CUDA_COMPILER_PATH.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 603 | default_clang_path = which('clang') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 604 | ask_clang_path = ('Please specify which clang should be used as device and ' |
| 605 | 'host compiler. [Default is %s]: ') % default_clang_path |
| 606 | |
| 607 | while True: |
| 608 | clang_cuda_compiler_path = get_from_env_or_user_or_default( |
| 609 | environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path, |
| 610 | default_clang_path) |
| 611 | if os.path.exists(clang_cuda_compiler_path): |
| 612 | break |
| 613 | |
| 614 | # Reset and retry |
| 615 | print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path) |
| 616 | environ_cp['CLANG_CUDA_COMPILER_PATH'] = '' |
| 617 | |
| 618 | # Set CLANG_CUDA_COMPILER_PATH |
| 619 | environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path |
| 620 | write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH', |
| 621 | clang_cuda_compiler_path) |
| 622 | |
| 623 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 624 | def prompt_loop_or_load_from_env(environ_cp, |
| 625 | var_name, |
| 626 | var_default, |
| 627 | ask_for_var, |
| 628 | check_success, |
| 629 | error_msg, |
| 630 | suppress_default_error=False, |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 631 | resolve_symlinks=False, |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 632 | n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS): |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 633 | """Loop over user prompts for an ENV param until receiving a valid response. |
| 634 | |
| 635 | For the env param var_name, read from the environment or verify user input |
| 636 | until receiving valid input. When done, set var_name in the environ_cp to its |
| 637 | new value. |
| 638 | |
| 639 | Args: |
| 640 | environ_cp: (Dict) copy of the os.environ. |
| 641 | var_name: (String) string for name of environment variable, e.g. "TF_MYVAR". |
| 642 | var_default: (String) default value string. |
| 643 | ask_for_var: (String) string for how to ask for user input. |
| 644 | check_success: (Function) function that takes one argument and returns a |
| 645 | boolean. Should return True if the value provided is considered valid. May |
| 646 | contain a complex error message if error_msg does not provide enough |
| 647 | information. In that case, set suppress_default_error to True. |
| 648 | error_msg: (String) String with one and only one '%s'. Formatted with each |
| 649 | invalid response upon check_success(input) failure. |
| 650 | suppress_default_error: (Bool) Suppress the above error message in favor of |
| 651 | one from the check_success function. |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 652 | resolve_symlinks: (Bool) Translate symbolic links into the real filepath. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 653 | n_ask_attempts: (Integer) Number of times to query for valid input before |
| 654 | raising an error and quitting. |
| 655 | |
| 656 | Returns: |
| 657 | [String] The value of var_name after querying for input. |
| 658 | |
| 659 | Raises: |
| 660 | UserInputError: if a query has been attempted n_ask_attempts times without |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 661 | success, assume that the user has made a scripting error, and will |
| 662 | continue to provide invalid input. Raise the error to avoid infinitely |
| 663 | looping. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 664 | """ |
| 665 | default = environ_cp.get(var_name) or var_default |
| 666 | full_query = '%s [Default is %s]: ' % ( |
| 667 | ask_for_var, |
| 668 | default, |
| 669 | ) |
| 670 | |
| 671 | for _ in range(n_ask_attempts): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 672 | val = get_from_env_or_user_or_default(environ_cp, var_name, full_query, |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 673 | default) |
| 674 | if check_success(val): |
| 675 | break |
| 676 | if not suppress_default_error: |
| 677 | print(error_msg % val) |
| 678 | environ_cp[var_name] = '' |
| 679 | else: |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 680 | raise UserInputError('Invalid %s setting was provided %d times in a row. ' |
| 681 | 'Assuming to be a scripting mistake.' % |
| 682 | (var_name, n_ask_attempts)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 683 | |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 684 | if resolve_symlinks and os.path.islink(val): |
| 685 | val = os.path.realpath(val) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 686 | environ_cp[var_name] = val |
| 687 | return val |
| 688 | |
| 689 | |
| 690 | def create_android_ndk_rule(environ_cp): |
| 691 | """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule.""" |
| 692 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 693 | default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' % |
| 694 | environ_cp['APPDATA']) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 695 | elif is_macos(): |
| 696 | default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 697 | else: |
| 698 | default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 699 | |
| 700 | def valid_ndk_path(path): |
| 701 | return (os.path.exists(path) and |
| 702 | os.path.exists(os.path.join(path, 'source.properties'))) |
| 703 | |
| 704 | android_ndk_home_path = prompt_loop_or_load_from_env( |
| 705 | environ_cp, |
| 706 | var_name='ANDROID_NDK_HOME', |
| 707 | var_default=default_ndk_path, |
| 708 | ask_for_var='Please specify the home path of the Android NDK to use.', |
| 709 | check_success=valid_ndk_path, |
| 710 | error_msg=('The path %s or its child file "source.properties" ' |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 711 | 'does not exist.')) |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 712 | write_action_env_to_bazelrc('ANDROID_NDK_HOME', android_ndk_home_path) |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 713 | write_action_env_to_bazelrc( |
| 714 | 'ANDROID_NDK_API_LEVEL', |
| 715 | get_ndk_api_level(environ_cp, android_ndk_home_path)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 716 | |
| 717 | |
| 718 | def create_android_sdk_rule(environ_cp): |
| 719 | """Set Android variables and write Android SDK WORKSPACE rule.""" |
| 720 | if is_windows() or is_cygwin(): |
| 721 | default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA']) |
| 722 | elif is_macos(): |
Shashi Shekhar | c0ff0cc | 2018-07-17 09:00:24 -0700 | [diff] [blame] | 723 | default_sdk_path = '%s/library/Android/Sdk' % environ_cp['HOME'] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 724 | else: |
| 725 | default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME'] |
| 726 | |
| 727 | def valid_sdk_path(path): |
| 728 | return (os.path.exists(path) and |
| 729 | os.path.exists(os.path.join(path, 'platforms')) and |
| 730 | os.path.exists(os.path.join(path, 'build-tools'))) |
| 731 | |
| 732 | android_sdk_home_path = prompt_loop_or_load_from_env( |
| 733 | environ_cp, |
| 734 | var_name='ANDROID_SDK_HOME', |
| 735 | var_default=default_sdk_path, |
| 736 | ask_for_var='Please specify the home path of the Android SDK to use.', |
| 737 | check_success=valid_sdk_path, |
| 738 | error_msg=('Either %s does not exist, or it does not contain the ' |
| 739 | 'subdirectories "platforms" and "build-tools".')) |
| 740 | |
| 741 | platforms = os.path.join(android_sdk_home_path, 'platforms') |
| 742 | api_levels = sorted(os.listdir(platforms)) |
| 743 | api_levels = [x.replace('android-', '') for x in api_levels] |
| 744 | |
| 745 | def valid_api_level(api_level): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 746 | return os.path.exists( |
| 747 | os.path.join(android_sdk_home_path, 'platforms', |
| 748 | 'android-' + api_level)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 749 | |
| 750 | android_api_level = prompt_loop_or_load_from_env( |
| 751 | environ_cp, |
| 752 | var_name='ANDROID_API_LEVEL', |
| 753 | var_default=api_levels[-1], |
| 754 | ask_for_var=('Please specify the Android SDK API level to use. ' |
| 755 | '[Available levels: %s]') % api_levels, |
| 756 | check_success=valid_api_level, |
| 757 | error_msg='Android-%s is not present in the SDK path.') |
| 758 | |
| 759 | build_tools = os.path.join(android_sdk_home_path, 'build-tools') |
| 760 | versions = sorted(os.listdir(build_tools)) |
| 761 | |
| 762 | def valid_build_tools(version): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 763 | return os.path.exists( |
| 764 | os.path.join(android_sdk_home_path, 'build-tools', version)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 765 | |
| 766 | android_build_tools_version = prompt_loop_or_load_from_env( |
| 767 | environ_cp, |
| 768 | var_name='ANDROID_BUILD_TOOLS_VERSION', |
| 769 | var_default=versions[-1], |
| 770 | ask_for_var=('Please specify an Android build tools version to use. ' |
| 771 | '[Available versions: %s]') % versions, |
| 772 | check_success=valid_build_tools, |
| 773 | error_msg=('The selected SDK does not have build-tools version %s ' |
| 774 | 'available.')) |
| 775 | |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 776 | write_action_env_to_bazelrc('ANDROID_BUILD_TOOLS_VERSION', |
| 777 | android_build_tools_version) |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 778 | write_action_env_to_bazelrc('ANDROID_SDK_API_LEVEL', android_api_level) |
| 779 | write_action_env_to_bazelrc('ANDROID_SDK_HOME', android_sdk_home_path) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 780 | |
| 781 | |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 782 | def get_ndk_api_level(environ_cp, android_ndk_home_path): |
| 783 | """Gets the appropriate NDK API level to use for the provided Android NDK path.""" |
| 784 | |
| 785 | # First check to see if we're using a blessed version of the NDK. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 786 | properties_path = '%s/source.properties' % android_ndk_home_path |
| 787 | if is_windows() or is_cygwin(): |
| 788 | properties_path = cygpath(properties_path) |
| 789 | with open(properties_path, 'r') as f: |
| 790 | filedata = f.read() |
| 791 | |
| 792 | revision = re.search(r'Pkg.Revision = (\d+)', filedata) |
| 793 | if revision: |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 794 | ndk_version = revision.group(1) |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 795 | else: |
| 796 | raise Exception('Unable to parse NDK revision.') |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 797 | if int(ndk_version) not in _SUPPORTED_ANDROID_NDK_VERSIONS: |
| 798 | print('WARNING: The NDK version in %s is %s, which is not ' |
| 799 | 'supported by Bazel (officially supported versions: %s). Please use ' |
| 800 | 'another version. Compiling Android targets may result in confusing ' |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 801 | 'errors.\n' % |
| 802 | (android_ndk_home_path, ndk_version, _SUPPORTED_ANDROID_NDK_VERSIONS)) |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 803 | |
| 804 | # Now grab the NDK API level to use. Note that this is different from the |
| 805 | # SDK API level, as the NDK API level is effectively the *min* target SDK |
| 806 | # version. |
| 807 | platforms = os.path.join(android_ndk_home_path, 'platforms') |
| 808 | api_levels = sorted(os.listdir(platforms)) |
| 809 | api_levels = [ |
| 810 | x.replace('android-', '') for x in api_levels if 'android-' in x |
| 811 | ] |
| 812 | |
| 813 | def valid_api_level(api_level): |
| 814 | return os.path.exists( |
| 815 | os.path.join(android_ndk_home_path, 'platforms', |
| 816 | 'android-' + api_level)) |
| 817 | |
| 818 | android_ndk_api_level = prompt_loop_or_load_from_env( |
| 819 | environ_cp, |
| 820 | var_name='ANDROID_NDK_API_LEVEL', |
Jared Duke | a1309f4 | 2019-11-15 17:09:35 -0800 | [diff] [blame] | 821 | var_default='21', # 21 is required for ARM64 support. |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 822 | ask_for_var=('Please specify the (min) Android NDK API level to use. ' |
| 823 | '[Available levels: %s]') % api_levels, |
| 824 | check_success=valid_api_level, |
| 825 | error_msg='Android-%s is not present in the NDK path.') |
| 826 | |
| 827 | return android_ndk_api_level |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 828 | |
| 829 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 830 | def set_gcc_host_compiler_path(environ_cp): |
| 831 | """Set GCC_HOST_COMPILER_PATH.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 832 | default_gcc_host_compiler_path = which('gcc') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 833 | cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH') |
| 834 | |
| 835 | if os.path.islink(cuda_bin_symlink): |
| 836 | # os.readlink is only available in linux |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 837 | default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 838 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 839 | gcc_host_compiler_path = prompt_loop_or_load_from_env( |
| 840 | environ_cp, |
| 841 | var_name='GCC_HOST_COMPILER_PATH', |
| 842 | var_default=default_gcc_host_compiler_path, |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 843 | ask_for_var='Please specify which gcc should be used by nvcc as the host compiler.', |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 844 | check_success=os.path.exists, |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 845 | resolve_symlinks=True, |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 846 | error_msg='Invalid gcc path. %s cannot be found.', |
| 847 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 848 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 849 | write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path) |
| 850 | |
| 851 | |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 852 | def reformat_version_sequence(version_str, sequence_count): |
| 853 | """Reformat the version string to have the given number of sequences. |
| 854 | |
| 855 | For example: |
| 856 | Given (7, 2) -> 7.0 |
| 857 | (7.0.1, 2) -> 7.0 |
| 858 | (5, 1) -> 5 |
| 859 | (5.0.3.2, 1) -> 5 |
| 860 | |
| 861 | Args: |
| 862 | version_str: String, the version string. |
| 863 | sequence_count: int, an integer. |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 864 | |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 865 | Returns: |
| 866 | string, reformatted version string. |
| 867 | """ |
| 868 | v = version_str.split('.') |
| 869 | if len(v) < sequence_count: |
| 870 | v = v + (['0'] * (sequence_count - len(v))) |
| 871 | |
| 872 | return '.'.join(v[:sequence_count]) |
| 873 | |
| 874 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 875 | def set_tf_cuda_paths(environ_cp): |
| 876 | """Set TF_CUDA_PATHS.""" |
| 877 | ask_cuda_paths = ( |
| 878 | 'Please specify the comma-separated list of base paths to look for CUDA ' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 879 | 'libraries and headers. [Leave empty to use the default]: ') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 880 | tf_cuda_paths = get_from_env_or_user_or_default(environ_cp, 'TF_CUDA_PATHS', |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 881 | ask_cuda_paths, '') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 882 | if tf_cuda_paths: |
| 883 | environ_cp['TF_CUDA_PATHS'] = tf_cuda_paths |
| 884 | |
| 885 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 886 | def set_tf_cuda_version(environ_cp): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 887 | """Set TF_CUDA_VERSION.""" |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 888 | ask_cuda_version = ( |
| 889 | 'Please specify the CUDA SDK version you want to use. ' |
| 890 | '[Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 891 | tf_cuda_version = get_from_env_or_user_or_default(environ_cp, |
| 892 | 'TF_CUDA_VERSION', |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 893 | ask_cuda_version, |
| 894 | _DEFAULT_CUDA_VERSION) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 895 | environ_cp['TF_CUDA_VERSION'] = tf_cuda_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 896 | |
| 897 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 898 | def set_tf_cudnn_version(environ_cp): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 899 | """Set TF_CUDNN_VERSION.""" |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 900 | ask_cudnn_version = ( |
| 901 | 'Please specify the cuDNN version you want to use. ' |
| 902 | '[Leave empty to default to cuDNN %s]: ') % _DEFAULT_CUDNN_VERSION |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 903 | tf_cudnn_version = get_from_env_or_user_or_default(environ_cp, |
| 904 | 'TF_CUDNN_VERSION', |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 905 | ask_cudnn_version, |
| 906 | _DEFAULT_CUDNN_VERSION) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 907 | environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 908 | |
| 909 | |
Mingxing Tan | 1e7b0e4 | 2018-06-28 19:13:20 -0700 | [diff] [blame] | 910 | def is_cuda_compatible(lib, cuda_ver, cudnn_ver): |
| 911 | """Check compatibility between given library and cudnn/cudart libraries.""" |
| 912 | ldd_bin = which('ldd') or '/usr/bin/ldd' |
| 913 | ldd_out = run_shell([ldd_bin, lib], True) |
| 914 | ldd_out = ldd_out.split(os.linesep) |
| 915 | cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$') |
| 916 | cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$') |
| 917 | cudnn = None |
| 918 | cudart = None |
| 919 | cudnn_ok = True # assume no cudnn dependency by default |
| 920 | cuda_ok = True # assume no cuda dependency by default |
| 921 | for line in ldd_out: |
| 922 | if 'libcudnn.so' in line: |
| 923 | cudnn = cudnn_pattern.search(line) |
| 924 | cudnn_ok = False |
| 925 | elif 'libcudart.so' in line: |
| 926 | cudart = cuda_pattern.search(line) |
| 927 | cuda_ok = False |
| 928 | if cudnn and len(cudnn.group(1)): |
| 929 | cudnn = convert_version_to_int(cudnn.group(1)) |
| 930 | if cudart and len(cudart.group(1)): |
| 931 | cudart = convert_version_to_int(cudart.group(1)) |
| 932 | if cudnn is not None: |
| 933 | cudnn_ok = (cudnn == cudnn_ver) |
| 934 | if cudart is not None: |
| 935 | cuda_ok = (cudart == cuda_ver) |
| 936 | return cudnn_ok and cuda_ok |
| 937 | |
| 938 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 939 | def set_tf_tensorrt_version(environ_cp): |
| 940 | """Set TF_TENSORRT_VERSION.""" |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 941 | if not is_linux(): |
| 942 | raise ValueError('Currently TensorRT is only supported on Linux platform.') |
| 943 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 944 | if not int(environ_cp.get('TF_NEED_TENSORRT', False)): |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 945 | return |
| 946 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 947 | ask_tensorrt_version = ( |
| 948 | 'Please specify the TensorRT version you want to use. ' |
Guangda Lai | faa93ac | 2019-10-23 15:07:12 -0700 | [diff] [blame] | 949 | '[Leave empty to default to TensorRT %s]: ') % _DEFAULT_TENSORRT_VERSION |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 950 | tf_tensorrt_version = get_from_env_or_user_or_default( |
| 951 | environ_cp, 'TF_TENSORRT_VERSION', ask_tensorrt_version, |
| 952 | _DEFAULT_TENSORRT_VERSION) |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 953 | environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 954 | |
| 955 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 956 | def set_tf_nccl_version(environ_cp): |
| 957 | """Set TF_NCCL_VERSION.""" |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 958 | if not is_linux(): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 959 | raise ValueError('Currently NCCL is only supported on Linux platform.') |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 960 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 961 | if 'TF_NCCL_VERSION' in environ_cp: |
| 962 | return |
| 963 | |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 964 | ask_nccl_version = ( |
A. Unique TensorFlower | 53faa31 | 2018-10-05 08:46:54 -0700 | [diff] [blame] | 965 | 'Please specify the locally installed NCCL version you want to use. ' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 966 | '[Leave empty to use http://github.com/nvidia/nccl]: ') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 967 | tf_nccl_version = get_from_env_or_user_or_default(environ_cp, |
| 968 | 'TF_NCCL_VERSION', |
| 969 | ask_nccl_version, '') |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 970 | environ_cp['TF_NCCL_VERSION'] = tf_nccl_version |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 971 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 972 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 973 | def get_native_cuda_compute_capabilities(environ_cp): |
| 974 | """Get native cuda compute capabilities. |
| 975 | |
| 976 | Args: |
| 977 | environ_cp: copy of the os.environ. |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 978 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 979 | Returns: |
| 980 | string of native cuda compute capabilities, separated by comma. |
| 981 | """ |
| 982 | device_query_bin = os.path.join( |
| 983 | environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery') |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 984 | if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK): |
| 985 | try: |
| 986 | output = run_shell(device_query_bin).split('\n') |
| 987 | pattern = re.compile('[0-9]*\\.[0-9]*') |
| 988 | output = [pattern.search(x) for x in output if 'Capability' in x] |
| 989 | output = ','.join(x.group() for x in output if x is not None) |
| 990 | except subprocess.CalledProcessError: |
| 991 | output = '' |
| 992 | else: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 993 | output = '' |
| 994 | return output |
| 995 | |
| 996 | |
| 997 | def set_tf_cuda_compute_capabilities(environ_cp): |
| 998 | """Set TF_CUDA_COMPUTE_CAPABILITIES.""" |
| 999 | while True: |
| 1000 | native_cuda_compute_capabilities = get_native_cuda_compute_capabilities( |
| 1001 | environ_cp) |
| 1002 | if not native_cuda_compute_capabilities: |
| 1003 | default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES |
| 1004 | else: |
| 1005 | default_cuda_compute_capabilities = native_cuda_compute_capabilities |
| 1006 | |
| 1007 | ask_cuda_compute_capabilities = ( |
| 1008 | 'Please specify a list of comma-separated ' |
P Sudeepam | 5209356 | 2019-02-17 17:34:01 +0530 | [diff] [blame] | 1009 | 'CUDA compute capabilities you want to ' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1010 | 'build with.\nYou can find the compute ' |
| 1011 | 'capability of your device at: ' |
| 1012 | 'https://developer.nvidia.com/cuda-gpus.\nPlease' |
| 1013 | ' note that each additional compute ' |
| 1014 | 'capability significantly increases your ' |
P Sudeepam | 5209356 | 2019-02-17 17:34:01 +0530 | [diff] [blame] | 1015 | 'build time and binary size, and that ' |
| 1016 | 'TensorFlow only supports compute ' |
P Sudeepam | 765ceda | 2019-02-17 17:39:08 +0530 | [diff] [blame] | 1017 | 'capabilities >= 3.5 [Default is: %s]: ' % |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1018 | default_cuda_compute_capabilities) |
| 1019 | tf_cuda_compute_capabilities = get_from_env_or_user_or_default( |
| 1020 | environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES', |
| 1021 | ask_cuda_compute_capabilities, default_cuda_compute_capabilities) |
| 1022 | # Check whether all capabilities from the input is valid |
| 1023 | all_valid = True |
Maciej | d0f5bc1 | 2018-04-30 22:30:58 -0500 | [diff] [blame] | 1024 | # Remove all whitespace characters before splitting the string |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 1025 | # that users may insert by accident, as this will result in error |
Maciej | d0f5bc1 | 2018-04-30 22:30:58 -0500 | [diff] [blame] | 1026 | tf_cuda_compute_capabilities = ''.join(tf_cuda_compute_capabilities.split()) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1027 | for compute_capability in tf_cuda_compute_capabilities.split(','): |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1028 | m = re.match('[0-9]+.[0-9]+', compute_capability) |
| 1029 | if not m: |
Austin Anderson | 32202dc | 2019-02-19 10:46:27 -0800 | [diff] [blame] | 1030 | print('Invalid compute capability: %s' % compute_capability) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1031 | all_valid = False |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1032 | else: |
P Sudeepam | 5209356 | 2019-02-17 17:34:01 +0530 | [diff] [blame] | 1033 | ver = float(m.group(0)) |
A. Unique TensorFlower | 8dc2d0e | 2019-03-12 01:41:05 -0700 | [diff] [blame] | 1034 | if ver < 3.0: |
| 1035 | print('ERROR: TensorFlow only supports CUDA compute capabilities 3.0 ' |
Austin Anderson | 32202dc | 2019-02-19 10:46:27 -0800 | [diff] [blame] | 1036 | 'and higher. Please re-specify the list of compute ' |
| 1037 | 'capabilities excluding version %s.' % ver) |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1038 | all_valid = False |
A. Unique TensorFlower | 8dc2d0e | 2019-03-12 01:41:05 -0700 | [diff] [blame] | 1039 | if ver < 3.5: |
| 1040 | print('WARNING: XLA does not support CUDA compute capabilities ' |
| 1041 | 'lower than 3.5. Disable XLA when running on older GPUs.') |
P Sudeepam | 765ceda | 2019-02-17 17:39:08 +0530 | [diff] [blame] | 1042 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1043 | if all_valid: |
| 1044 | break |
| 1045 | |
| 1046 | # Reset and Retry |
| 1047 | environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = '' |
| 1048 | |
| 1049 | # Set TF_CUDA_COMPUTE_CAPABILITIES |
| 1050 | environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities |
| 1051 | write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES', |
| 1052 | tf_cuda_compute_capabilities) |
| 1053 | |
| 1054 | |
| 1055 | def set_other_cuda_vars(environ_cp): |
| 1056 | """Set other CUDA related variables.""" |
A. Unique TensorFlower | ab39198 | 2018-07-11 04:52:49 -0700 | [diff] [blame] | 1057 | # If CUDA is enabled, always use GPU during build and test. |
| 1058 | if environ_cp.get('TF_CUDA_CLANG') == '1': |
| 1059 | write_to_bazelrc('build --config=cuda_clang') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1060 | else: |
A. Unique TensorFlower | ab39198 | 2018-07-11 04:52:49 -0700 | [diff] [blame] | 1061 | write_to_bazelrc('build --config=cuda') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1062 | |
| 1063 | |
| 1064 | def set_host_cxx_compiler(environ_cp): |
| 1065 | """Set HOST_CXX_COMPILER.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1066 | default_cxx_host_compiler = which('g++') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1067 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1068 | host_cxx_compiler = prompt_loop_or_load_from_env( |
| 1069 | environ_cp, |
| 1070 | var_name='HOST_CXX_COMPILER', |
| 1071 | var_default=default_cxx_host_compiler, |
| 1072 | ask_for_var=('Please specify which C++ compiler should be used as the ' |
| 1073 | 'host C++ compiler.'), |
| 1074 | check_success=os.path.exists, |
| 1075 | error_msg='Invalid C++ compiler path. %s cannot be found.', |
| 1076 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1077 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1078 | write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler) |
| 1079 | |
| 1080 | |
| 1081 | def set_host_c_compiler(environ_cp): |
| 1082 | """Set HOST_C_COMPILER.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1083 | default_c_host_compiler = which('gcc') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1084 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1085 | host_c_compiler = prompt_loop_or_load_from_env( |
| 1086 | environ_cp, |
| 1087 | var_name='HOST_C_COMPILER', |
| 1088 | var_default=default_c_host_compiler, |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1089 | ask_for_var=('Please specify which C compiler should be used as the host ' |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1090 | 'C compiler.'), |
| 1091 | check_success=os.path.exists, |
| 1092 | error_msg='Invalid C compiler path. %s cannot be found.', |
| 1093 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1094 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1095 | write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler) |
| 1096 | |
| 1097 | |
| 1098 | def set_computecpp_toolkit_path(environ_cp): |
| 1099 | """Set COMPUTECPP_TOOLKIT_PATH.""" |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1100 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1101 | def toolkit_exists(toolkit_path): |
| 1102 | """Check if a computecpp toolkit path is valid.""" |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1103 | if is_linux(): |
| 1104 | sycl_rt_lib_path = 'lib/libComputeCpp.so' |
| 1105 | else: |
| 1106 | sycl_rt_lib_path = '' |
| 1107 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1108 | sycl_rt_lib_path_full = os.path.join(toolkit_path, sycl_rt_lib_path) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1109 | exists = os.path.exists(sycl_rt_lib_path_full) |
| 1110 | if not exists: |
| 1111 | print('Invalid SYCL %s library path. %s cannot be found' % |
| 1112 | (_TF_OPENCL_VERSION, sycl_rt_lib_path_full)) |
| 1113 | return exists |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1114 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1115 | computecpp_toolkit_path = prompt_loop_or_load_from_env( |
| 1116 | environ_cp, |
| 1117 | var_name='COMPUTECPP_TOOLKIT_PATH', |
| 1118 | var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH, |
| 1119 | ask_for_var=( |
| 1120 | 'Please specify the location where ComputeCpp for SYCL %s is ' |
| 1121 | 'installed.' % _TF_OPENCL_VERSION), |
| 1122 | check_success=toolkit_exists, |
| 1123 | error_msg='Invalid SYCL compiler path. %s cannot be found.', |
| 1124 | suppress_default_error=True) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1125 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1126 | write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH', |
| 1127 | computecpp_toolkit_path) |
| 1128 | |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1129 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1130 | def set_trisycl_include_dir(environ_cp): |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1131 | """Set TRISYCL_INCLUDE_DIR.""" |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 1132 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1133 | ask_trisycl_include_dir = ('Please specify the location of the triSYCL ' |
| 1134 | 'include directory. (Use --config=sycl_trisycl ' |
| 1135 | 'when building with Bazel) ' |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1136 | '[Default is %s]: ') % ( |
| 1137 | _DEFAULT_TRISYCL_INCLUDE_DIR) |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 1138 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1139 | while True: |
| 1140 | trisycl_include_dir = get_from_env_or_user_or_default( |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1141 | environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir, |
| 1142 | _DEFAULT_TRISYCL_INCLUDE_DIR) |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1143 | if os.path.exists(trisycl_include_dir): |
| 1144 | break |
| 1145 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1146 | print('Invalid triSYCL include directory, %s cannot be found' % |
| 1147 | (trisycl_include_dir)) |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1148 | |
| 1149 | # Set TRISYCL_INCLUDE_DIR |
| 1150 | environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1151 | write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', trisycl_include_dir) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1152 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1153 | |
A. Unique TensorFlower | e9797fd | 2019-03-22 11:02:40 -0700 | [diff] [blame] | 1154 | def system_specific_test_config(env): |
A. Unique TensorFlower | 7bd8637 | 2019-03-21 15:19:30 -0700 | [diff] [blame] | 1155 | """Add default build and test flags required for TF tests to bazelrc.""" |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1156 | write_to_bazelrc('test --flaky_test_attempts=3') |
| 1157 | write_to_bazelrc('test --test_size_filters=small,medium') |
frreiss | 8b013a7 | 2019-11-20 11:36:27 -0800 | [diff] [blame] | 1158 | |
| 1159 | # Each instance of --test_tag_filters or --build_tag_filters overrides all |
| 1160 | # previous instances, so we need to build up a complete list and write a |
| 1161 | # single list of filters for the .bazelrc file. |
| 1162 | |
| 1163 | # Filters to use with both --test_tag_filters and --build_tag_filters |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame^] | 1164 | test_and_build_filters = ['-benchmark-test', '-no_oss'] |
| 1165 | # Additional filters for --test_tag_filters beyond those in |
| 1166 | # test_and_build_filters |
| 1167 | test_only_filters = ['-oss_serial'] |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1168 | if is_windows(): |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame^] | 1169 | test_and_build_filters.append('-no_windows') |
Guangda Lai | bcd701a | 2019-03-12 21:04:51 -0700 | [diff] [blame] | 1170 | if env.get('TF_NEED_CUDA', None) == '1': |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame^] | 1171 | test_and_build_filters += ['-no_windows_gpu', '-no_gpu'] |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1172 | else: |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame^] | 1173 | test_and_build_filters.append('-gpu') |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1174 | elif is_macos(): |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame^] | 1175 | test_and_build_filters += ['-gpu', '-nomac', '-no_mac'] |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1176 | elif is_linux(): |
Guangda Lai | bcd701a | 2019-03-12 21:04:51 -0700 | [diff] [blame] | 1177 | if env.get('TF_NEED_CUDA', None) == '1': |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame^] | 1178 | test_and_build_filters.append('-no_gpu') |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1179 | write_to_bazelrc('test --test_env=LD_LIBRARY_PATH') |
| 1180 | else: |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame^] | 1181 | test_and_build_filters.append('-gpu') |
frreiss | 8b013a7 | 2019-11-20 11:36:27 -0800 | [diff] [blame] | 1182 | write_to_bazelrc('test --test_tag_filters=%s' |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame^] | 1183 | % ','.join(test_and_build_filters + test_only_filters)) |
frreiss | 8b013a7 | 2019-11-20 11:36:27 -0800 | [diff] [blame] | 1184 | write_to_bazelrc('test --build_tag_filters=%s' |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame^] | 1185 | % ','.join(test_and_build_filters)) |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1186 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1187 | |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1188 | def set_system_libs_flag(environ_cp): |
| 1189 | syslibs = environ_cp.get('TF_SYSTEM_LIBS', '') |
TensorFlower Gardener | 61a8720 | 2018-10-01 12:25:39 -0700 | [diff] [blame] | 1190 | if syslibs: |
Jason Zaman | 5fc39bd | 2018-09-16 01:38:55 +0800 | [diff] [blame] | 1191 | if ',' in syslibs: |
| 1192 | syslibs = ','.join(sorted(syslibs.split(','))) |
| 1193 | else: |
| 1194 | syslibs = ','.join(sorted(syslibs.split())) |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1195 | write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs) |
| 1196 | |
Jason Zaman | 5fc39bd | 2018-09-16 01:38:55 +0800 | [diff] [blame] | 1197 | if 'PREFIX' in environ_cp: |
| 1198 | write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX']) |
| 1199 | if 'LIBDIR' in environ_cp: |
| 1200 | write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR']) |
| 1201 | if 'INCLUDEDIR' in environ_cp: |
| 1202 | write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR']) |
| 1203 | |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1204 | |
Austin Anderson | 6d7c376 | 2019-11-19 17:04:19 -0800 | [diff] [blame] | 1205 | def is_reduced_optimize_huge_functions_available(environ_cp): |
| 1206 | """Check to see if the system supports /d2ReducedOptimizeHugeFunctions. |
| 1207 | |
| 1208 | The above compiler flag is a new compiler flag introduced to the Visual Studio |
| 1209 | compiler in version 16.4 (available in Visual Studio 2019, Preview edition |
| 1210 | only, as of 2019-11-19). TensorFlow needs this flag to massively reduce |
| 1211 | compile times, but until 16.4 is officially released, we can't depend on it. |
| 1212 | |
| 1213 | See also https://groups.google.com/a/tensorflow.org/g/build/c/SsW98Eo7l3o |
| 1214 | |
| 1215 | Because it's very annoying to check this manually (to check the MSVC installed |
| 1216 | versions, you need to use the registry, and it's not clear if Bazel will be |
| 1217 | using that install version anyway), we expect enviroments who know they may |
| 1218 | use this flag to export TF_VC_VERSION=16.4 |
| 1219 | |
| 1220 | TODO(angerson, gunan): Remove this function when TensorFlow's minimum VS |
| 1221 | version is upgraded to 16.4. |
| 1222 | |
| 1223 | Arguments: |
| 1224 | environ_cp: Environment of the current execution |
| 1225 | |
| 1226 | Returns: |
| 1227 | boolean, whether or not /d2ReducedOptimizeHugeFunctions is available on this |
| 1228 | machine. |
| 1229 | """ |
| 1230 | return float(environ_cp.get('TF_VC_VERSION', '0')) >= 16.4 |
| 1231 | |
| 1232 | |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1233 | def set_windows_build_flags(environ_cp): |
| 1234 | """Set Windows specific build options.""" |
| 1235 | # The non-monolithic build is not supported yet |
| 1236 | write_to_bazelrc('build --config monolithic') |
| 1237 | # Suppress warning messages |
| 1238 | write_to_bazelrc('build --copt=-w --host_copt=-w') |
Loo Rong Jie | 31f10d2 | 2019-02-02 10:03:20 +0800 | [diff] [blame] | 1239 | # Fix winsock2.h conflicts |
TensorFlower Gardener | 345cccf | 2019-02-28 15:22:59 -0800 | [diff] [blame] | 1240 | write_to_bazelrc( |
Justin Lebar | 7b57c5a | 2019-04-30 12:24:29 -0700 | [diff] [blame] | 1241 | 'build --copt=-DWIN32_LEAN_AND_MEAN --host_copt=-DWIN32_LEAN_AND_MEAN ' |
| 1242 | '--copt=-DNOGDI --host_copt=-DNOGDI') |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1243 | # Output more verbose information when something goes wrong |
| 1244 | write_to_bazelrc('build --verbose_failures') |
| 1245 | # The host and target platforms are the same in Windows build. So we don't |
| 1246 | # have to distinct them. This avoids building the same targets twice. |
| 1247 | write_to_bazelrc('build --distinct_host_configuration=false') |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1248 | |
Austin Anderson | 6d7c376 | 2019-11-19 17:04:19 -0800 | [diff] [blame] | 1249 | if is_reduced_optimize_huge_functions_available(environ_cp): |
| 1250 | write_to_bazelrc( |
| 1251 | 'build --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions' |
| 1252 | ) |
| 1253 | |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1254 | if get_var( |
| 1255 | environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline', |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1256 | True, ('Would you like to override eigen strong inline for some C++ ' |
| 1257 | 'compilation to reduce the compilation time?'), |
| 1258 | 'Eigen strong inline overridden.', 'Not overriding eigen strong inline, ' |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1259 | 'some compilations could take more than 20 mins.'): |
| 1260 | # Due to a known MSVC compiler issue |
| 1261 | # https://github.com/tensorflow/tensorflow/issues/10521 |
| 1262 | # Overriding eigen strong inline speeds up the compiling of |
| 1263 | # conv_grad_ops_3d.cc and conv_ops_3d.cc by 20 minutes, |
| 1264 | # but this also hurts the performance. Let users decide what they want. |
| 1265 | write_to_bazelrc('build --define=override_eigen_strong_inline=true') |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1266 | |
A. Unique TensorFlower | 061c359 | 2017-11-13 14:21:04 -0800 | [diff] [blame] | 1267 | |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1268 | def config_info_line(name, help_text): |
| 1269 | """Helper function to print formatted help text for Bazel config options.""" |
| 1270 | print('\t--config=%-12s\t# %s' % (name, help_text)) |
| 1271 | |
| 1272 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1273 | def configure_ios(): |
| 1274 | """Configures TensorFlow for iOS builds. |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1275 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1276 | This function will only be executed if `is_macos()` is true. |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1277 | """ |
| 1278 | if not is_macos(): |
| 1279 | return |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1280 | for filepath in APPLE_BAZEL_FILES: |
| 1281 | existing_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath + '.apple') |
| 1282 | renamed_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath) |
| 1283 | symlink_force(existing_filepath, renamed_filepath) |
| 1284 | for filepath in IOS_FILES: |
| 1285 | filename = os.path.basename(filepath) |
| 1286 | new_filepath = os.path.join(_TF_WORKSPACE_ROOT, filename) |
| 1287 | symlink_force(filepath, new_filepath) |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1288 | |
| 1289 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1290 | def validate_cuda_config(environ_cp): |
| 1291 | """Run find_cuda_config.py and return cuda_toolkit_path, or None.""" |
| 1292 | |
| 1293 | def maybe_encode_env(env): |
| 1294 | """Encodes unicode in env to str on Windows python 2.x.""" |
| 1295 | if not is_windows() or sys.version_info[0] != 2: |
| 1296 | return env |
| 1297 | for k, v in env.items(): |
| 1298 | if isinstance(k, unicode): |
| 1299 | k = k.encode('ascii') |
| 1300 | if isinstance(v, unicode): |
| 1301 | v = v.encode('ascii') |
| 1302 | env[k] = v |
| 1303 | return env |
| 1304 | |
| 1305 | cuda_libraries = ['cuda', 'cudnn'] |
| 1306 | if is_linux(): |
A. Unique TensorFlower | 28fc9cc | 2019-05-01 14:17:54 -0700 | [diff] [blame] | 1307 | if int(environ_cp.get('TF_NEED_TENSORRT', False)): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1308 | cuda_libraries.append('tensorrt') |
A. Unique TensorFlower | b2e7f67 | 2019-04-30 09:20:36 -0700 | [diff] [blame] | 1309 | if environ_cp.get('TF_NCCL_VERSION', None): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1310 | cuda_libraries.append('nccl') |
| 1311 | |
| 1312 | proc = subprocess.Popen( |
| 1313 | [environ_cp['PYTHON_BIN_PATH'], 'third_party/gpus/find_cuda_config.py'] + |
| 1314 | cuda_libraries, |
| 1315 | stdout=subprocess.PIPE, |
| 1316 | env=maybe_encode_env(environ_cp)) |
| 1317 | |
| 1318 | if proc.wait(): |
| 1319 | # Errors from find_cuda_config.py were sent to stderr. |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 1320 | print('Asking for detailed CUDA configuration...\n') |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1321 | return False |
| 1322 | |
| 1323 | config = dict( |
| 1324 | tuple(line.decode('ascii').rstrip().split(': ')) for line in proc.stdout) |
| 1325 | |
| 1326 | print('Found CUDA %s in:' % config['cuda_version']) |
| 1327 | print(' %s' % config['cuda_library_dir']) |
| 1328 | print(' %s' % config['cuda_include_dir']) |
| 1329 | |
| 1330 | print('Found cuDNN %s in:' % config['cudnn_version']) |
| 1331 | print(' %s' % config['cudnn_library_dir']) |
| 1332 | print(' %s' % config['cudnn_include_dir']) |
| 1333 | |
| 1334 | if 'tensorrt_version' in config: |
| 1335 | print('Found TensorRT %s in:' % config['tensorrt_version']) |
| 1336 | print(' %s' % config['tensorrt_library_dir']) |
| 1337 | print(' %s' % config['tensorrt_include_dir']) |
| 1338 | |
| 1339 | if config.get('nccl_version', None): |
| 1340 | print('Found NCCL %s in:' % config['nccl_version']) |
| 1341 | print(' %s' % config['nccl_library_dir']) |
| 1342 | print(' %s' % config['nccl_include_dir']) |
| 1343 | |
| 1344 | print('\n') |
| 1345 | |
| 1346 | environ_cp['CUDA_TOOLKIT_PATH'] = config['cuda_toolkit_path'] |
| 1347 | return True |
| 1348 | |
| 1349 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1350 | def main(): |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1351 | global _TF_WORKSPACE_ROOT |
| 1352 | global _TF_BAZELRC |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 1353 | global _TF_CURRENT_BAZEL_VERSION |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1354 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1355 | parser = argparse.ArgumentParser() |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1356 | parser.add_argument( |
| 1357 | '--workspace', |
| 1358 | type=str, |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1359 | default=os.path.abspath(os.path.dirname(__file__)), |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1360 | help='The absolute path to your active Bazel workspace.') |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1361 | args = parser.parse_args() |
| 1362 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1363 | _TF_WORKSPACE_ROOT = args.workspace |
| 1364 | _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME) |
| 1365 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1366 | # Make a copy of os.environ to be clear when functions and getting and setting |
| 1367 | # environment variables. |
| 1368 | environ_cp = dict(os.environ) |
| 1369 | |
Mark Daoust | 44000ad | 2019-06-18 09:26:26 -0700 | [diff] [blame] | 1370 | current_bazel_version = check_bazel_version(_TF_MIN_BAZEL_VERSION, |
| 1371 | _TF_MAX_BAZEL_VERSION) |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 1372 | _TF_CURRENT_BAZEL_VERSION = convert_version_to_int(current_bazel_version) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1373 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1374 | reset_tf_configure_bazelrc() |
Yun Peng | 03e63a2 | 2018-11-07 11:18:53 +0100 | [diff] [blame] | 1375 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1376 | cleanup_makefile() |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 1377 | setup_python(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1378 | |
| 1379 | if is_windows(): |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1380 | environ_cp['TF_NEED_OPENCL_SYCL'] = '0' |
| 1381 | environ_cp['TF_NEED_COMPUTECPP'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1382 | environ_cp['TF_NEED_OPENCL'] = '0' |
| 1383 | environ_cp['TF_CUDA_CLANG'] = '0' |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 1384 | environ_cp['TF_NEED_TENSORRT'] = '0' |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1385 | # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on |
| 1386 | # Windows. |
| 1387 | environ_cp['TF_DOWNLOAD_CLANG'] = '0' |
A. Unique TensorFlower | 6e97fb3 | 2018-07-16 14:07:29 -0700 | [diff] [blame] | 1388 | environ_cp['TF_NEED_MPI'] = '0' |
| 1389 | environ_cp['TF_SET_ANDROID_WORKSPACE'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1390 | |
| 1391 | if is_macos(): |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 1392 | environ_cp['TF_NEED_TENSORRT'] = '0' |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1393 | else: |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1394 | environ_cp['TF_CONFIGURE_IOS'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1395 | |
Sanjoy Das | f3fab7c | 2019-06-20 11:19:00 -0700 | [diff] [blame] | 1396 | xla_enabled_by_default = is_linux() or is_macos() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1397 | set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support', |
Grzegorz Pawelczak | ec82efd | 2018-10-09 15:03:46 +0100 | [diff] [blame] | 1398 | xla_enabled_by_default, 'xla') |
Gunhan Gulsoy | 3da0dff | 2018-09-26 11:55:50 -0700 | [diff] [blame] | 1399 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1400 | set_action_env_var( |
| 1401 | environ_cp, |
| 1402 | 'TF_NEED_OPENCL_SYCL', |
| 1403 | 'OpenCL SYCL', |
| 1404 | False, |
| 1405 | bazel_config_name='sycl') |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1406 | if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1407 | set_host_cxx_compiler(environ_cp) |
| 1408 | set_host_c_compiler(environ_cp) |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1409 | set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True) |
| 1410 | if environ_cp.get('TF_NEED_COMPUTECPP') == '1': |
| 1411 | set_computecpp_toolkit_path(environ_cp) |
| 1412 | else: |
| 1413 | set_trisycl_include_dir(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1414 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1415 | set_action_env_var( |
| 1416 | environ_cp, 'TF_NEED_ROCM', 'ROCm', False, bazel_config_name='rocm') |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1417 | if (environ_cp.get('TF_NEED_ROCM') == '1' and |
TensorFlower Gardener | 62e6016 | 2018-09-27 10:22:55 -0700 | [diff] [blame] | 1418 | 'LD_LIBRARY_PATH' in environ_cp and |
| 1419 | environ_cp.get('LD_LIBRARY_PATH') != '1'): |
| 1420 | write_action_env_to_bazelrc('LD_LIBRARY_PATH', |
| 1421 | environ_cp.get('LD_LIBRARY_PATH')) |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1422 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1423 | environ_cp['TF_NEED_CUDA'] = str( |
| 1424 | int(get_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False))) |
A. Unique TensorFlower | 24cbb2a | 2017-09-08 07:45:44 -0700 | [diff] [blame] | 1425 | if (environ_cp.get('TF_NEED_CUDA') == '1' and |
| 1426 | 'TF_CUDA_CONFIG_REPO' not in environ_cp): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1427 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1428 | set_action_env_var( |
| 1429 | environ_cp, |
| 1430 | 'TF_NEED_TENSORRT', |
| 1431 | 'TensorRT', |
| 1432 | False, |
| 1433 | bazel_config_name='tensorrt') |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1434 | |
| 1435 | environ_save = dict(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1436 | for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1437 | |
| 1438 | if validate_cuda_config(environ_cp): |
| 1439 | cuda_env_names = [ |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1440 | 'TF_CUDA_VERSION', |
| 1441 | 'TF_CUBLAS_VERSION', |
| 1442 | 'TF_CUDNN_VERSION', |
| 1443 | 'TF_TENSORRT_VERSION', |
| 1444 | 'TF_NCCL_VERSION', |
| 1445 | 'TF_CUDA_PATHS', |
A. Unique TensorFlower | b2e7f67 | 2019-04-30 09:20:36 -0700 | [diff] [blame] | 1446 | # Items below are for backwards compatibility when not using |
| 1447 | # TF_CUDA_PATHS. |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1448 | 'CUDA_TOOLKIT_PATH', |
| 1449 | 'CUDNN_INSTALL_PATH', |
| 1450 | 'NCCL_INSTALL_PATH', |
| 1451 | 'NCCL_HDR_PATH', |
| 1452 | 'TENSORRT_INSTALL_PATH' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1453 | ] |
A. Unique TensorFlower | b2e7f67 | 2019-04-30 09:20:36 -0700 | [diff] [blame] | 1454 | # Note: set_action_env_var above already writes to bazelrc. |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1455 | for name in cuda_env_names: |
| 1456 | if name in environ_cp: |
| 1457 | write_action_env_to_bazelrc(name, environ_cp[name]) |
| 1458 | break |
| 1459 | |
| 1460 | # Restore settings changed below if CUDA config could not be validated. |
| 1461 | environ_cp = dict(environ_save) |
| 1462 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1463 | set_tf_cuda_version(environ_cp) |
| 1464 | set_tf_cudnn_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1465 | if is_linux(): |
| 1466 | set_tf_tensorrt_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1467 | set_tf_nccl_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1468 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1469 | set_tf_cuda_paths(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1470 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1471 | else: |
| 1472 | raise UserInputError( |
| 1473 | 'Invalid CUDA setting were provided %d ' |
| 1474 | 'times in a row. Assuming to be a scripting mistake.' % |
| 1475 | _DEFAULT_PROMPT_ASK_ATTEMPTS) |
| 1476 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1477 | set_tf_cuda_compute_capabilities(environ_cp) |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 1478 | if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get( |
| 1479 | 'LD_LIBRARY_PATH') != '1': |
| 1480 | write_action_env_to_bazelrc('LD_LIBRARY_PATH', |
| 1481 | environ_cp.get('LD_LIBRARY_PATH')) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1482 | |
| 1483 | set_tf_cuda_clang(environ_cp) |
| 1484 | if environ_cp.get('TF_CUDA_CLANG') == '1': |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1485 | # Ask whether we should download the clang toolchain. |
| 1486 | set_tf_download_clang(environ_cp) |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 1487 | if environ_cp.get('TF_DOWNLOAD_CLANG') != '1': |
| 1488 | # Set up which clang we should use as the cuda / host compiler. |
| 1489 | set_clang_cuda_compiler_path(environ_cp) |
Ilya Biryukov | 1c3d02e | 2018-09-04 03:09:52 -0700 | [diff] [blame] | 1490 | else: |
| 1491 | # Use downloaded LLD for linking. |
| 1492 | write_to_bazelrc('build:cuda_clang --config=download_clang_use_lld') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1493 | else: |
| 1494 | # Set up which gcc nvcc should use as the host compiler |
| 1495 | # No need to set this on Windows |
| 1496 | if not is_windows(): |
| 1497 | set_gcc_host_compiler_path(environ_cp) |
| 1498 | set_other_cuda_vars(environ_cp) |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1499 | else: |
| 1500 | # CUDA not required. Ask whether we should download the clang toolchain and |
| 1501 | # use it for the CPU build. |
| 1502 | set_tf_download_clang(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1503 | |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1504 | # SYCL / ROCm / CUDA are mutually exclusive. |
| 1505 | # At most 1 GPU platform can be configured. |
| 1506 | gpu_platform_count = 0 |
| 1507 | if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': |
| 1508 | gpu_platform_count += 1 |
| 1509 | if environ_cp.get('TF_NEED_ROCM') == '1': |
| 1510 | gpu_platform_count += 1 |
| 1511 | if environ_cp.get('TF_NEED_CUDA') == '1': |
| 1512 | gpu_platform_count += 1 |
| 1513 | if gpu_platform_count >= 2: |
| 1514 | raise UserInputError('SYCL / CUDA / ROCm are mututally exclusive. ' |
| 1515 | 'At most 1 GPU platform can be configured.') |
| 1516 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1517 | set_cc_opt_flags(environ_cp) |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1518 | set_system_libs_flag(environ_cp) |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1519 | if is_windows(): |
| 1520 | set_windows_build_flags(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1521 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1522 | if get_var(environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', False, |
| 1523 | ('Would you like to interactively configure ./WORKSPACE for ' |
| 1524 | 'Android builds?'), 'Searching for NDK and SDK installations.', |
| 1525 | 'Not configuring the WORKSPACE for Android builds.'): |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 1526 | create_android_ndk_rule(environ_cp) |
| 1527 | create_android_sdk_rule(environ_cp) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1528 | |
A. Unique TensorFlower | e9797fd | 2019-03-22 11:02:40 -0700 | [diff] [blame] | 1529 | system_specific_test_config(os.environ) |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1530 | |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 1531 | set_action_env_var(environ_cp, 'TF_CONFIGURE_IOS', 'iOS', False) |
| 1532 | if environ_cp.get('TF_CONFIGURE_IOS') == '1': |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1533 | configure_ios() |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1534 | |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1535 | print('Preconfigured Bazel build configs. You can use any of the below by ' |
| 1536 | 'adding "--config=<>" to your build command. See .bazelrc for more ' |
| 1537 | 'details.') |
| 1538 | config_info_line('mkl', 'Build with MKL support.') |
| 1539 | config_info_line('monolithic', 'Config for mostly static monolithic build.') |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1540 | config_info_line('ngraph', 'Build with Intel nGraph support.') |
A. Unique TensorFlower | a6bf9c8 | 2019-02-26 10:08:35 -0800 | [diff] [blame] | 1541 | config_info_line('numa', 'Build with NUMA support.') |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 1542 | config_info_line( |
| 1543 | 'dynamic_kernels', |
| 1544 | '(Experimental) Build kernels into separate shared objects.') |
Anna R | aedf674 | 2019-06-12 11:30:51 -0700 | [diff] [blame] | 1545 | config_info_line('v2', 'Build TensorFlow 2.x instead of 1.x.') |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1546 | |
| 1547 | print('Preconfigured Bazel build configs to DISABLE default on features:') |
| 1548 | config_info_line('noaws', 'Disable AWS S3 filesystem support.') |
| 1549 | config_info_line('nogcp', 'Disable GCP support.') |
| 1550 | config_info_line('nohdfs', 'Disable HDFS support.') |
Gunhan Gulsoy | eea8168 | 2018-11-26 16:51:23 -0800 | [diff] [blame] | 1551 | config_info_line('nonccl', 'Disable NVIDIA NCCL support.') |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1552 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1553 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1554 | if __name__ == '__main__': |
| 1555 | main() |