A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | # ============================================================================== |
| 15 | """configure script to get build parameters from user.""" |
| 16 | |
| 17 | from __future__ import absolute_import |
| 18 | from __future__ import division |
| 19 | from __future__ import print_function |
| 20 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 21 | import argparse |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 22 | import errno |
| 23 | import os |
| 24 | import platform |
| 25 | import re |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 26 | import subprocess |
| 27 | import sys |
| 28 | |
Andrew Selle | c9885ea | 2017-11-06 09:37:03 -0800 | [diff] [blame] | 29 | # pylint: disable=g-import-not-at-top |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 30 | try: |
| 31 | from shutil import which |
| 32 | except ImportError: |
| 33 | from distutils.spawn import find_executable as which |
Andrew Selle | c9885ea | 2017-11-06 09:37:03 -0800 | [diff] [blame] | 34 | # pylint: enable=g-import-not-at-top |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 35 | |
Christian Sigg | 8dff6b9 | 2019-12-09 04:05:46 -0800 | [diff] [blame] | 36 | _DEFAULT_CUDA_VERSION = '10' |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 37 | _DEFAULT_CUDNN_VERSION = '7' |
Guangda Lai | faa93ac | 2019-10-23 15:07:12 -0700 | [diff] [blame] | 38 | _DEFAULT_TENSORRT_VERSION = '6' |
Smit Hinsu | fe7d1d9 | 2018-07-14 13:16:58 -0700 | [diff] [blame] | 39 | _DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 40 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 41 | _TF_OPENCL_VERSION = '1.2' |
| 42 | _DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp' |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 43 | _DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include' |
A. Unique TensorFlower | 82820ef | 2018-11-12 13:22:13 -0800 | [diff] [blame] | 44 | _SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15, 16, 17, 18] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 45 | |
| 46 | _DEFAULT_PROMPT_ASK_ATTEMPTS = 10 |
| 47 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 48 | _TF_BAZELRC_FILENAME = '.tf_configure.bazelrc' |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 49 | _TF_WORKSPACE_ROOT = '' |
| 50 | _TF_BAZELRC = '' |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 51 | _TF_CURRENT_BAZEL_VERSION = None |
Jonathan DEKHTIAR | e23ef99 | 2020-02-19 17:32:07 -0800 | [diff] [blame] | 52 | _TF_MIN_BAZEL_VERSION = '2.0.0' |
Gunhan Gulsoy | 2f65c24 | 2020-04-14 21:07:52 -0700 | [diff] [blame] | 53 | _TF_MAX_BAZEL_VERSION = '3.99.0' |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 54 | |
Jason Furmanek | 7c23415 | 2018-09-26 04:44:12 +0000 | [diff] [blame] | 55 | NCCL_LIB_PATHS = [ |
TensorFlower Gardener | ea5c529 | 2018-10-01 17:28:26 -0700 | [diff] [blame] | 56 | 'lib64/', 'lib/powerpc64le-linux-gnu/', 'lib/x86_64-linux-gnu/', '' |
Jason Furmanek | 7c23415 | 2018-09-26 04:44:12 +0000 | [diff] [blame] | 57 | ] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 58 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 59 | # List of files to configure when building Bazel on Apple platforms. |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 60 | APPLE_BAZEL_FILES = [ |
A. Unique TensorFlower | 6a059c0 | 2019-04-05 14:28:27 -0700 | [diff] [blame] | 61 | 'tensorflow/lite/experimental/ios/BUILD', |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 62 | 'tensorflow/lite/experimental/objc/BUILD', |
YoungSeok Yoon | 7089aaa | 2019-11-11 21:36:17 -0800 | [diff] [blame] | 63 | 'tensorflow/lite/experimental/swift/BUILD', |
| 64 | 'tensorflow/lite/tools/benchmark/experimental/ios/BUILD' |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 65 | ] |
| 66 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 67 | # List of files to move when building for iOS. |
| 68 | IOS_FILES = [ |
| 69 | 'tensorflow/lite/experimental/objc/TensorFlowLiteObjC.podspec', |
| 70 | 'tensorflow/lite/experimental/swift/TensorFlowLiteSwift.podspec', |
| 71 | ] |
| 72 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 73 | |
| 74 | class UserInputError(Exception): |
| 75 | pass |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 76 | |
| 77 | |
| 78 | def is_windows(): |
| 79 | return platform.system() == 'Windows' |
| 80 | |
| 81 | |
| 82 | def is_linux(): |
| 83 | return platform.system() == 'Linux' |
| 84 | |
| 85 | |
| 86 | def is_macos(): |
| 87 | return platform.system() == 'Darwin' |
| 88 | |
| 89 | |
| 90 | def is_ppc64le(): |
| 91 | return platform.machine() == 'ppc64le' |
| 92 | |
| 93 | |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 94 | def is_cygwin(): |
| 95 | return platform.system().startswith('CYGWIN_NT') |
| 96 | |
| 97 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 98 | def get_input(question): |
| 99 | try: |
| 100 | try: |
| 101 | answer = raw_input(question) |
| 102 | except NameError: |
| 103 | answer = input(question) # pylint: disable=bad-builtin |
| 104 | except EOFError: |
| 105 | answer = '' |
| 106 | return answer |
| 107 | |
| 108 | |
| 109 | def symlink_force(target, link_name): |
| 110 | """Force symlink, equivalent of 'ln -sf'. |
| 111 | |
| 112 | Args: |
| 113 | target: items to link to. |
| 114 | link_name: name of the link. |
| 115 | """ |
| 116 | try: |
| 117 | os.symlink(target, link_name) |
| 118 | except OSError as e: |
| 119 | if e.errno == errno.EEXIST: |
| 120 | os.remove(link_name) |
| 121 | os.symlink(target, link_name) |
| 122 | else: |
| 123 | raise e |
| 124 | |
| 125 | |
| 126 | def sed_in_place(filename, old, new): |
| 127 | """Replace old string with new string in file. |
| 128 | |
| 129 | Args: |
| 130 | filename: string for filename. |
| 131 | old: string to replace. |
| 132 | new: new string to replace to. |
| 133 | """ |
| 134 | with open(filename, 'r') as f: |
| 135 | filedata = f.read() |
| 136 | newdata = filedata.replace(old, new) |
| 137 | with open(filename, 'w') as f: |
| 138 | f.write(newdata) |
| 139 | |
| 140 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 141 | def write_to_bazelrc(line): |
| 142 | with open(_TF_BAZELRC, 'a') as f: |
| 143 | f.write(line + '\n') |
| 144 | |
| 145 | |
| 146 | def write_action_env_to_bazelrc(var_name, var): |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 147 | write_to_bazelrc('build --action_env {}="{}"'.format(var_name, str(var))) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 148 | |
| 149 | |
Wallyss Lima | 7237104 | 2019-10-03 12:55:30 -0300 | [diff] [blame] | 150 | def run_shell(cmd, allow_non_zero=False, stderr=None): |
| 151 | if stderr is None: |
| 152 | stderr = sys.stdout |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 153 | if allow_non_zero: |
| 154 | try: |
Wallyss Lima | 7237104 | 2019-10-03 12:55:30 -0300 | [diff] [blame] | 155 | output = subprocess.check_output(cmd, stderr=stderr) |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 156 | except subprocess.CalledProcessError as e: |
| 157 | output = e.output |
| 158 | else: |
Wallyss Lima | 7237104 | 2019-10-03 12:55:30 -0300 | [diff] [blame] | 159 | output = subprocess.check_output(cmd, stderr=stderr) |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 160 | return output.decode('UTF-8').strip() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 161 | |
| 162 | |
| 163 | def cygpath(path): |
| 164 | """Convert path from posix to windows.""" |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 165 | return os.path.abspath(path).replace('\\', '/') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 166 | |
| 167 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 168 | def get_python_path(environ_cp, python_bin_path): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 169 | """Get the python site package paths.""" |
| 170 | python_paths = [] |
| 171 | if environ_cp.get('PYTHONPATH'): |
| 172 | python_paths = environ_cp.get('PYTHONPATH').split(':') |
| 173 | try: |
Wallyss Lima | 484304e | 2019-10-03 12:58:20 -0300 | [diff] [blame] | 174 | stderr = open(os.devnull, 'wb') |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 175 | library_paths = run_shell([ |
| 176 | python_bin_path, '-c', |
| 177 | 'import site; print("\\n".join(site.getsitepackages()))' |
TensorFlower Gardener | 839a436 | 2019-12-30 02:23:29 -0800 | [diff] [blame] | 178 | ], |
| 179 | stderr=stderr).split('\n') |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 180 | except subprocess.CalledProcessError: |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 181 | library_paths = [ |
| 182 | run_shell([ |
| 183 | python_bin_path, '-c', |
| 184 | 'from distutils.sysconfig import get_python_lib;' |
| 185 | 'print(get_python_lib())' |
| 186 | ]) |
| 187 | ] |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 188 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 189 | all_paths = set(python_paths + library_paths) |
| 190 | |
| 191 | paths = [] |
| 192 | for path in all_paths: |
| 193 | if os.path.isdir(path): |
| 194 | paths.append(path) |
| 195 | return paths |
| 196 | |
| 197 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 198 | def get_python_major_version(python_bin_path): |
| 199 | """Get the python major version.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 200 | return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])']) |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 201 | |
| 202 | |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 203 | def setup_python(environ_cp): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 204 | """Setup python related env variables.""" |
| 205 | # Get PYTHON_BIN_PATH, default is the current running python. |
| 206 | default_python_bin_path = sys.executable |
| 207 | ask_python_bin_path = ('Please specify the location of python. [Default is ' |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 208 | '{}]: ').format(default_python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 209 | while True: |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 210 | python_bin_path = get_from_env_or_user_or_default(environ_cp, |
| 211 | 'PYTHON_BIN_PATH', |
| 212 | ask_python_bin_path, |
| 213 | default_python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 214 | # Check if the path is valid |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 215 | if os.path.isfile(python_bin_path) and os.access(python_bin_path, os.X_OK): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 216 | break |
| 217 | elif not os.path.exists(python_bin_path): |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 218 | print('Invalid python path: {} cannot be found.'.format(python_bin_path)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 219 | else: |
TensorFlower Gardener | 9784cf2 | 2020-05-12 15:05:26 -0700 | [diff] [blame] | 220 | print('{} is not executable. Is it the python binary?'.format( |
| 221 | python_bin_path)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 222 | environ_cp['PYTHON_BIN_PATH'] = '' |
| 223 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 224 | # Convert python path to Windows style before checking lib and version |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 225 | if is_windows() or is_cygwin(): |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 226 | python_bin_path = cygpath(python_bin_path) |
| 227 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 228 | # Get PYTHON_LIB_PATH |
| 229 | python_lib_path = environ_cp.get('PYTHON_LIB_PATH') |
| 230 | if not python_lib_path: |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 231 | python_lib_paths = get_python_path(environ_cp, python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 232 | if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1': |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 233 | python_lib_path = python_lib_paths[0] |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 234 | else: |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 235 | print('Found possible Python library paths:\n %s' % |
| 236 | '\n '.join(python_lib_paths)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 237 | default_python_lib_path = python_lib_paths[0] |
| 238 | python_lib_path = get_input( |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 239 | 'Please input the desired Python library path to use. ' |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 240 | 'Default is [{}]\n'.format(python_lib_paths[0])) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 241 | if not python_lib_path: |
| 242 | python_lib_path = default_python_lib_path |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 243 | environ_cp['PYTHON_LIB_PATH'] = python_lib_path |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 244 | |
A. Unique TensorFlower | cfccbdb | 2019-06-24 11:01:58 -0700 | [diff] [blame] | 245 | python_major_version = get_python_major_version(python_bin_path) |
| 246 | if python_major_version == '2': |
| 247 | write_to_bazelrc('build --host_force_python=PY2') |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 248 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 249 | # Convert python path to Windows style before writing into bazel.rc |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 250 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 251 | python_lib_path = cygpath(python_lib_path) |
| 252 | |
| 253 | # Set-up env variables used by python_configure.bzl |
| 254 | write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path) |
| 255 | write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path) |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 256 | write_to_bazelrc('build --python_path=\"{}"'.format(python_bin_path)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 257 | environ_cp['PYTHON_BIN_PATH'] = python_bin_path |
| 258 | |
William D. Irons | dcc76a5 | 2018-11-20 10:35:18 -0600 | [diff] [blame] | 259 | # If choosen python_lib_path is from a path specified in the PYTHONPATH |
| 260 | # variable, need to tell bazel to include PYTHONPATH |
| 261 | if environ_cp.get('PYTHONPATH'): |
| 262 | python_paths = environ_cp.get('PYTHONPATH').split(':') |
| 263 | if python_lib_path in python_paths: |
TensorFlower Gardener | 968cd18 | 2018-11-28 11:33:16 -0800 | [diff] [blame] | 264 | write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH')) |
William D. Irons | dcc76a5 | 2018-11-20 10:35:18 -0600 | [diff] [blame] | 265 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 266 | # Write tools/python_bin_path.sh |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 267 | with open( |
| 268 | os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'), |
| 269 | 'w') as f: |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 270 | f.write('export PYTHON_BIN_PATH="{}"'.format(python_bin_path)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 271 | |
| 272 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 273 | def reset_tf_configure_bazelrc(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 274 | """Reset file that contains customized config settings.""" |
| 275 | open(_TF_BAZELRC, 'w').close() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 276 | |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 277 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 278 | def cleanup_makefile(): |
| 279 | """Delete any leftover BUILD files from the Makefile build. |
| 280 | |
| 281 | These files could interfere with Bazel parsing. |
| 282 | """ |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 283 | makefile_download_dir = os.path.join(_TF_WORKSPACE_ROOT, 'tensorflow', |
| 284 | 'contrib', 'makefile', 'downloads') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 285 | if os.path.isdir(makefile_download_dir): |
| 286 | for root, _, filenames in os.walk(makefile_download_dir): |
| 287 | for f in filenames: |
| 288 | if f.endswith('BUILD'): |
| 289 | os.remove(os.path.join(root, f)) |
| 290 | |
| 291 | |
| 292 | def get_var(environ_cp, |
| 293 | var_name, |
| 294 | query_item, |
| 295 | enabled_by_default, |
| 296 | question=None, |
| 297 | yes_reply=None, |
| 298 | no_reply=None): |
| 299 | """Get boolean input from user. |
| 300 | |
| 301 | If var_name is not set in env, ask user to enable query_item or not. If the |
| 302 | response is empty, use the default. |
| 303 | |
| 304 | Args: |
| 305 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 306 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
| 307 | query_item: string for feature related to the variable, e.g. "CUDA for |
| 308 | Nvidia GPUs". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 309 | enabled_by_default: boolean for default behavior. |
| 310 | question: optional string for how to ask for user input. |
Michael Case | d90054e | 2018-02-07 14:36:00 -0800 | [diff] [blame] | 311 | yes_reply: optional string for reply when feature is enabled. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 312 | no_reply: optional string for reply when feature is disabled. |
| 313 | |
| 314 | Returns: |
| 315 | boolean value of the variable. |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 316 | |
| 317 | Raises: |
| 318 | UserInputError: if an environment variable is set, but it cannot be |
| 319 | interpreted as a boolean indicator, assume that the user has made a |
| 320 | scripting error, and will continue to provide invalid input. |
| 321 | Raise the error to avoid infinitely looping. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 322 | """ |
| 323 | if not question: |
TensorFlower Gardener | 9784cf2 | 2020-05-12 15:05:26 -0700 | [diff] [blame] | 324 | question = 'Do you wish to build TensorFlow with {} support?'.format( |
| 325 | query_item) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 326 | if not yes_reply: |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 327 | yes_reply = '{} support will be enabled for TensorFlow.'.format(query_item) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 328 | if not no_reply: |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 329 | no_reply = 'No {}'.format(yes_reply) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 330 | |
| 331 | yes_reply += '\n' |
| 332 | no_reply += '\n' |
| 333 | |
| 334 | if enabled_by_default: |
| 335 | question += ' [Y/n]: ' |
| 336 | else: |
| 337 | question += ' [y/N]: ' |
| 338 | |
| 339 | var = environ_cp.get(var_name) |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 340 | if var is not None: |
| 341 | var_content = var.strip().lower() |
| 342 | true_strings = ('1', 't', 'true', 'y', 'yes') |
| 343 | false_strings = ('0', 'f', 'false', 'n', 'no') |
| 344 | if var_content in true_strings: |
| 345 | var = True |
| 346 | elif var_content in false_strings: |
| 347 | var = False |
| 348 | else: |
| 349 | raise UserInputError( |
| 350 | 'Environment variable %s must be set as a boolean indicator.\n' |
| 351 | 'The following are accepted as TRUE : %s.\n' |
| 352 | 'The following are accepted as FALSE: %s.\n' |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 353 | 'Current value is %s.' % |
| 354 | (var_name, ', '.join(true_strings), ', '.join(false_strings), var)) |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 355 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 356 | while var is None: |
| 357 | user_input_origin = get_input(question) |
| 358 | user_input = user_input_origin.strip().lower() |
| 359 | if user_input == 'y': |
| 360 | print(yes_reply) |
| 361 | var = True |
| 362 | elif user_input == 'n': |
| 363 | print(no_reply) |
| 364 | var = False |
| 365 | elif not user_input: |
| 366 | if enabled_by_default: |
| 367 | print(yes_reply) |
| 368 | var = True |
| 369 | else: |
| 370 | print(no_reply) |
| 371 | var = False |
| 372 | else: |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 373 | print('Invalid selection: {}'.format(user_input_origin)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 374 | return var |
| 375 | |
| 376 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 377 | def set_build_var(environ_cp, |
| 378 | var_name, |
| 379 | query_item, |
| 380 | option_name, |
| 381 | enabled_by_default, |
| 382 | bazel_config_name=None): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 383 | """Set if query_item will be enabled for the build. |
| 384 | |
| 385 | Ask user if query_item will be enabled. Default is used if no input is given. |
| 386 | Set subprocess environment variable and write to .bazelrc if enabled. |
| 387 | |
| 388 | Args: |
| 389 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 390 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
| 391 | query_item: string for feature related to the variable, e.g. "CUDA for |
| 392 | Nvidia GPUs". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 393 | option_name: string for option to define in .bazelrc. |
| 394 | enabled_by_default: boolean for default behavior. |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 395 | bazel_config_name: Name for Bazel --config argument to enable build feature. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 396 | """ |
| 397 | |
| 398 | var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default))) |
| 399 | environ_cp[var_name] = var |
| 400 | if var == '1': |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 401 | write_to_bazelrc('build:%s --define %s=true' % |
| 402 | (bazel_config_name, option_name)) |
Yifei Feng | ec451f5 | 2018-10-05 12:53:50 -0700 | [diff] [blame] | 403 | write_to_bazelrc('build --config=%s' % bazel_config_name) |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 404 | elif bazel_config_name is not None: |
| 405 | # TODO(mikecase): Migrate all users of configure.py to use --config Bazel |
| 406 | # options and not to set build configs through environment variables. |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 407 | write_to_bazelrc('build:%s --define %s=true' % |
| 408 | (bazel_config_name, option_name)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 409 | |
| 410 | |
| 411 | def set_action_env_var(environ_cp, |
| 412 | var_name, |
| 413 | query_item, |
| 414 | enabled_by_default, |
| 415 | question=None, |
| 416 | yes_reply=None, |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 417 | no_reply=None, |
| 418 | bazel_config_name=None): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 419 | """Set boolean action_env variable. |
| 420 | |
| 421 | Ask user if query_item will be enabled. Default is used if no input is given. |
| 422 | Set environment variable and write to .bazelrc. |
| 423 | |
| 424 | Args: |
| 425 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 426 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
| 427 | query_item: string for feature related to the variable, e.g. "CUDA for |
| 428 | Nvidia GPUs". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 429 | enabled_by_default: boolean for default behavior. |
| 430 | question: optional string for how to ask for user input. |
Michael Case | d90054e | 2018-02-07 14:36:00 -0800 | [diff] [blame] | 431 | yes_reply: optional string for reply when feature is enabled. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 432 | no_reply: optional string for reply when feature is disabled. |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 433 | bazel_config_name: adding config to .bazelrc instead of action_env. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 434 | """ |
| 435 | var = int( |
| 436 | get_var(environ_cp, var_name, query_item, enabled_by_default, question, |
| 437 | yes_reply, no_reply)) |
| 438 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 439 | if not bazel_config_name: |
| 440 | write_action_env_to_bazelrc(var_name, var) |
| 441 | elif var: |
| 442 | write_to_bazelrc('build --config=%s' % bazel_config_name) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 443 | environ_cp[var_name] = str(var) |
| 444 | |
| 445 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 446 | def convert_version_to_int(version): |
| 447 | """Convert a version number to a integer that can be used to compare. |
| 448 | |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 449 | Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The |
| 450 | 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored. |
| 451 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 452 | Args: |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 453 | version: a version to be converted |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 454 | |
| 455 | Returns: |
| 456 | An integer if converted successfully, otherwise return None. |
| 457 | """ |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 458 | version = version.split('-')[0] |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 459 | version_segments = version.split('.') |
Austin Anderson | 87ea41d | 2019-04-04 10:03:50 -0700 | [diff] [blame] | 460 | # Treat "0.24" as "0.24.0" |
| 461 | if len(version_segments) == 2: |
| 462 | version_segments.append('0') |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 463 | for seg in version_segments: |
| 464 | if not seg.isdigit(): |
| 465 | return None |
| 466 | |
| 467 | version_str = ''.join(['%03d' % int(seg) for seg in version_segments]) |
| 468 | return int(version_str) |
| 469 | |
| 470 | |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 471 | def check_bazel_version(min_version, max_version): |
| 472 | """Check installed bazel version is between min_version and max_version. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 473 | |
| 474 | Args: |
Mihai Maruseac | 4db860f | 2019-04-19 08:52:10 -0700 | [diff] [blame] | 475 | min_version: string for minimum bazel version (must exist!). |
| 476 | max_version: string for maximum bazel version (must exist!). |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 477 | |
| 478 | Returns: |
| 479 | The bazel version detected. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 480 | """ |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 481 | if which('bazel') is None: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 482 | print('Cannot find bazel. Please install bazel.') |
| 483 | sys.exit(0) |
| 484 | |
A. Unique TensorFlower | 56a99d6 | 2020-05-08 09:28:28 -0700 | [diff] [blame] | 485 | stderr = open(os.devnull, 'wb') |
| 486 | curr_version = run_shell(['bazel', '--version'], |
| 487 | allow_non_zero = True, |
| 488 | stderr = stderr) |
| 489 | if curr_version.startswith('bazel '): |
| 490 | curr_version = curr_version.split('bazel ')[1] |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 491 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 492 | min_version_int = convert_version_to_int(min_version) |
| 493 | curr_version_int = convert_version_to_int(curr_version) |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 494 | max_version_int = convert_version_to_int(max_version) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 495 | |
| 496 | # Check if current bazel version can be detected properly. |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 497 | if not curr_version_int: |
| 498 | print('WARNING: current bazel installation is not a release version.') |
| 499 | print('Make sure you are running at least bazel %s' % min_version) |
| 500 | return curr_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 501 | |
Michael Case | d94271a | 2017-08-22 17:26:52 -0700 | [diff] [blame] | 502 | print('You have bazel %s installed.' % curr_version) |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 503 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 504 | if curr_version_int < min_version_int: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 505 | print('Please upgrade your bazel installation to version %s or higher to ' |
| 506 | 'build TensorFlow!' % min_version) |
Jason Zaman | b41761c | 2018-10-14 11:28:53 +0800 | [diff] [blame] | 507 | sys.exit(1) |
TensorFlower Gardener | 78c246b | 2018-12-13 12:37:42 -0800 | [diff] [blame] | 508 | if (curr_version_int > max_version_int and |
| 509 | 'TF_IGNORE_MAX_BAZEL_VERSION' not in os.environ): |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 510 | print('Please downgrade your bazel installation to version %s or lower to ' |
Mihai Maruseac | e0963c4 | 2018-12-20 14:27:40 -0800 | [diff] [blame] | 511 | 'build TensorFlow! To downgrade: download the installer for the old ' |
| 512 | 'version (from https://github.com/bazelbuild/bazel/releases) then ' |
| 513 | 'run the installer.' % max_version) |
Jason Zaman | b41761c | 2018-10-14 11:28:53 +0800 | [diff] [blame] | 514 | sys.exit(1) |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 515 | return curr_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 516 | |
| 517 | |
| 518 | def set_cc_opt_flags(environ_cp): |
| 519 | """Set up architecture-dependent optimization flags. |
| 520 | |
| 521 | Also append CC optimization flags to bazel.rc.. |
| 522 | |
| 523 | Args: |
| 524 | environ_cp: copy of the os.environ. |
| 525 | """ |
| 526 | if is_ppc64le(): |
| 527 | # gcc on ppc64le does not support -march, use mcpu instead |
| 528 | default_cc_opt_flags = '-mcpu=native' |
A. Unique TensorFlower | 1bba94a | 2018-04-04 15:45:20 -0700 | [diff] [blame] | 529 | elif is_windows(): |
| 530 | default_cc_opt_flags = '/arch:AVX' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 531 | else: |
Justin Lebar | 9ef04f5 | 2018-10-10 18:52:45 -0700 | [diff] [blame] | 532 | default_cc_opt_flags = '-march=native -Wno-sign-compare' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 533 | question = ('Please specify optimization flags to use during compilation when' |
| 534 | ' bazel option "--config=opt" is specified [Default is %s]: ' |
| 535 | ) % default_cc_opt_flags |
| 536 | cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS', |
| 537 | question, default_cc_opt_flags) |
| 538 | for opt in cc_opt_flags.split(): |
Michael Case | 0017742 | 2017-11-10 13:14:03 -0800 | [diff] [blame] | 539 | write_to_bazelrc('build:opt --copt=%s' % opt) |
| 540 | # It should be safe on the same build host. |
A. Unique TensorFlower | 1bba94a | 2018-04-04 15:45:20 -0700 | [diff] [blame] | 541 | if not is_ppc64le() and not is_windows(): |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 542 | write_to_bazelrc('build:opt --host_copt=-march=native') |
Michael Case | bb3355d | 2017-11-09 08:46:31 -0800 | [diff] [blame] | 543 | write_to_bazelrc('build:opt --define with_default_optimizations=true') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 544 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 545 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 546 | def set_tf_cuda_clang(environ_cp): |
| 547 | """set TF_CUDA_CLANG action_env. |
| 548 | |
| 549 | Args: |
| 550 | environ_cp: copy of the os.environ. |
| 551 | """ |
| 552 | question = 'Do you want to use clang as CUDA compiler?' |
| 553 | yes_reply = 'Clang will be used as CUDA compiler.' |
| 554 | no_reply = 'nvcc will be used as CUDA compiler.' |
| 555 | set_action_env_var( |
| 556 | environ_cp, |
| 557 | 'TF_CUDA_CLANG', |
| 558 | None, |
| 559 | False, |
| 560 | question=question, |
| 561 | yes_reply=yes_reply, |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 562 | no_reply=no_reply, |
| 563 | bazel_config_name='cuda_clang') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 564 | |
| 565 | |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 566 | def set_tf_download_clang(environ_cp): |
| 567 | """Set TF_DOWNLOAD_CLANG action_env.""" |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 568 | question = 'Do you wish to download a fresh release of clang? (Experimental)' |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 569 | yes_reply = 'Clang will be downloaded and used to compile tensorflow.' |
| 570 | no_reply = 'Clang will not be downloaded.' |
| 571 | set_action_env_var( |
| 572 | environ_cp, |
| 573 | 'TF_DOWNLOAD_CLANG', |
| 574 | None, |
| 575 | False, |
| 576 | question=question, |
| 577 | yes_reply=yes_reply, |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 578 | no_reply=no_reply, |
| 579 | bazel_config_name='download_clang') |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 580 | |
| 581 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 582 | def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var, |
| 583 | var_default): |
| 584 | """Get var_name either from env, or user or default. |
| 585 | |
| 586 | If var_name has been set as environment variable, use the preset value, else |
| 587 | ask for user input. If no input is provided, the default is used. |
| 588 | |
| 589 | Args: |
| 590 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 591 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 592 | ask_for_var: string for how to ask for user input. |
| 593 | var_default: default value string. |
| 594 | |
| 595 | Returns: |
| 596 | string value for var_name |
| 597 | """ |
| 598 | var = environ_cp.get(var_name) |
| 599 | if not var: |
| 600 | var = get_input(ask_for_var) |
Michael Case | d94271a | 2017-08-22 17:26:52 -0700 | [diff] [blame] | 601 | print('\n') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 602 | if not var: |
| 603 | var = var_default |
| 604 | return var |
| 605 | |
| 606 | |
| 607 | def set_clang_cuda_compiler_path(environ_cp): |
| 608 | """Set CLANG_CUDA_COMPILER_PATH.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 609 | default_clang_path = which('clang') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 610 | ask_clang_path = ('Please specify which clang should be used as device and ' |
| 611 | 'host compiler. [Default is %s]: ') % default_clang_path |
| 612 | |
| 613 | while True: |
| 614 | clang_cuda_compiler_path = get_from_env_or_user_or_default( |
| 615 | environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path, |
| 616 | default_clang_path) |
| 617 | if os.path.exists(clang_cuda_compiler_path): |
| 618 | break |
| 619 | |
| 620 | # Reset and retry |
| 621 | print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path) |
| 622 | environ_cp['CLANG_CUDA_COMPILER_PATH'] = '' |
| 623 | |
| 624 | # Set CLANG_CUDA_COMPILER_PATH |
| 625 | environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path |
| 626 | write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH', |
| 627 | clang_cuda_compiler_path) |
| 628 | |
| 629 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 630 | def prompt_loop_or_load_from_env(environ_cp, |
| 631 | var_name, |
| 632 | var_default, |
| 633 | ask_for_var, |
| 634 | check_success, |
| 635 | error_msg, |
| 636 | suppress_default_error=False, |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 637 | resolve_symlinks=False, |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 638 | n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS): |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 639 | """Loop over user prompts for an ENV param until receiving a valid response. |
| 640 | |
| 641 | For the env param var_name, read from the environment or verify user input |
| 642 | until receiving valid input. When done, set var_name in the environ_cp to its |
| 643 | new value. |
| 644 | |
| 645 | Args: |
| 646 | environ_cp: (Dict) copy of the os.environ. |
| 647 | var_name: (String) string for name of environment variable, e.g. "TF_MYVAR". |
| 648 | var_default: (String) default value string. |
| 649 | ask_for_var: (String) string for how to ask for user input. |
| 650 | check_success: (Function) function that takes one argument and returns a |
| 651 | boolean. Should return True if the value provided is considered valid. May |
| 652 | contain a complex error message if error_msg does not provide enough |
| 653 | information. In that case, set suppress_default_error to True. |
| 654 | error_msg: (String) String with one and only one '%s'. Formatted with each |
| 655 | invalid response upon check_success(input) failure. |
| 656 | suppress_default_error: (Bool) Suppress the above error message in favor of |
| 657 | one from the check_success function. |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 658 | resolve_symlinks: (Bool) Translate symbolic links into the real filepath. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 659 | n_ask_attempts: (Integer) Number of times to query for valid input before |
| 660 | raising an error and quitting. |
| 661 | |
| 662 | Returns: |
| 663 | [String] The value of var_name after querying for input. |
| 664 | |
| 665 | Raises: |
| 666 | UserInputError: if a query has been attempted n_ask_attempts times without |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 667 | success, assume that the user has made a scripting error, and will |
| 668 | continue to provide invalid input. Raise the error to avoid infinitely |
| 669 | looping. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 670 | """ |
| 671 | default = environ_cp.get(var_name) or var_default |
| 672 | full_query = '%s [Default is %s]: ' % ( |
| 673 | ask_for_var, |
| 674 | default, |
| 675 | ) |
| 676 | |
| 677 | for _ in range(n_ask_attempts): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 678 | val = get_from_env_or_user_or_default(environ_cp, var_name, full_query, |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 679 | default) |
| 680 | if check_success(val): |
| 681 | break |
| 682 | if not suppress_default_error: |
| 683 | print(error_msg % val) |
| 684 | environ_cp[var_name] = '' |
| 685 | else: |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 686 | raise UserInputError('Invalid %s setting was provided %d times in a row. ' |
| 687 | 'Assuming to be a scripting mistake.' % |
| 688 | (var_name, n_ask_attempts)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 689 | |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 690 | if resolve_symlinks and os.path.islink(val): |
| 691 | val = os.path.realpath(val) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 692 | environ_cp[var_name] = val |
| 693 | return val |
| 694 | |
| 695 | |
| 696 | def create_android_ndk_rule(environ_cp): |
| 697 | """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule.""" |
| 698 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 699 | default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' % |
| 700 | environ_cp['APPDATA']) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 701 | elif is_macos(): |
| 702 | default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 703 | else: |
| 704 | default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 705 | |
| 706 | def valid_ndk_path(path): |
| 707 | return (os.path.exists(path) and |
| 708 | os.path.exists(os.path.join(path, 'source.properties'))) |
| 709 | |
| 710 | android_ndk_home_path = prompt_loop_or_load_from_env( |
| 711 | environ_cp, |
| 712 | var_name='ANDROID_NDK_HOME', |
| 713 | var_default=default_ndk_path, |
| 714 | ask_for_var='Please specify the home path of the Android NDK to use.', |
| 715 | check_success=valid_ndk_path, |
| 716 | error_msg=('The path %s or its child file "source.properties" ' |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 717 | 'does not exist.')) |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 718 | write_action_env_to_bazelrc('ANDROID_NDK_HOME', android_ndk_home_path) |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 719 | write_action_env_to_bazelrc( |
| 720 | 'ANDROID_NDK_API_LEVEL', |
| 721 | get_ndk_api_level(environ_cp, android_ndk_home_path)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 722 | |
| 723 | |
| 724 | def create_android_sdk_rule(environ_cp): |
| 725 | """Set Android variables and write Android SDK WORKSPACE rule.""" |
| 726 | if is_windows() or is_cygwin(): |
| 727 | default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA']) |
| 728 | elif is_macos(): |
Shashi Shekhar | c0ff0cc | 2018-07-17 09:00:24 -0700 | [diff] [blame] | 729 | default_sdk_path = '%s/library/Android/Sdk' % environ_cp['HOME'] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 730 | else: |
| 731 | default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME'] |
| 732 | |
| 733 | def valid_sdk_path(path): |
| 734 | return (os.path.exists(path) and |
| 735 | os.path.exists(os.path.join(path, 'platforms')) and |
| 736 | os.path.exists(os.path.join(path, 'build-tools'))) |
| 737 | |
| 738 | android_sdk_home_path = prompt_loop_or_load_from_env( |
| 739 | environ_cp, |
| 740 | var_name='ANDROID_SDK_HOME', |
| 741 | var_default=default_sdk_path, |
| 742 | ask_for_var='Please specify the home path of the Android SDK to use.', |
| 743 | check_success=valid_sdk_path, |
| 744 | error_msg=('Either %s does not exist, or it does not contain the ' |
| 745 | 'subdirectories "platforms" and "build-tools".')) |
| 746 | |
| 747 | platforms = os.path.join(android_sdk_home_path, 'platforms') |
| 748 | api_levels = sorted(os.listdir(platforms)) |
| 749 | api_levels = [x.replace('android-', '') for x in api_levels] |
| 750 | |
| 751 | def valid_api_level(api_level): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 752 | return os.path.exists( |
| 753 | os.path.join(android_sdk_home_path, 'platforms', |
| 754 | 'android-' + api_level)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 755 | |
| 756 | android_api_level = prompt_loop_or_load_from_env( |
| 757 | environ_cp, |
| 758 | var_name='ANDROID_API_LEVEL', |
| 759 | var_default=api_levels[-1], |
| 760 | ask_for_var=('Please specify the Android SDK API level to use. ' |
| 761 | '[Available levels: %s]') % api_levels, |
| 762 | check_success=valid_api_level, |
| 763 | error_msg='Android-%s is not present in the SDK path.') |
| 764 | |
| 765 | build_tools = os.path.join(android_sdk_home_path, 'build-tools') |
| 766 | versions = sorted(os.listdir(build_tools)) |
| 767 | |
| 768 | def valid_build_tools(version): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 769 | return os.path.exists( |
| 770 | os.path.join(android_sdk_home_path, 'build-tools', version)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 771 | |
| 772 | android_build_tools_version = prompt_loop_or_load_from_env( |
| 773 | environ_cp, |
| 774 | var_name='ANDROID_BUILD_TOOLS_VERSION', |
| 775 | var_default=versions[-1], |
| 776 | ask_for_var=('Please specify an Android build tools version to use. ' |
| 777 | '[Available versions: %s]') % versions, |
| 778 | check_success=valid_build_tools, |
| 779 | error_msg=('The selected SDK does not have build-tools version %s ' |
| 780 | 'available.')) |
| 781 | |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 782 | write_action_env_to_bazelrc('ANDROID_BUILD_TOOLS_VERSION', |
| 783 | android_build_tools_version) |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 784 | write_action_env_to_bazelrc('ANDROID_SDK_API_LEVEL', android_api_level) |
| 785 | write_action_env_to_bazelrc('ANDROID_SDK_HOME', android_sdk_home_path) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 786 | |
| 787 | |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 788 | def get_ndk_api_level(environ_cp, android_ndk_home_path): |
| 789 | """Gets the appropriate NDK API level to use for the provided Android NDK path.""" |
| 790 | |
| 791 | # First check to see if we're using a blessed version of the NDK. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 792 | properties_path = '%s/source.properties' % android_ndk_home_path |
| 793 | if is_windows() or is_cygwin(): |
| 794 | properties_path = cygpath(properties_path) |
| 795 | with open(properties_path, 'r') as f: |
| 796 | filedata = f.read() |
| 797 | |
| 798 | revision = re.search(r'Pkg.Revision = (\d+)', filedata) |
| 799 | if revision: |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 800 | ndk_version = revision.group(1) |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 801 | else: |
| 802 | raise Exception('Unable to parse NDK revision.') |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 803 | if int(ndk_version) not in _SUPPORTED_ANDROID_NDK_VERSIONS: |
| 804 | print('WARNING: The NDK version in %s is %s, which is not ' |
| 805 | 'supported by Bazel (officially supported versions: %s). Please use ' |
| 806 | 'another version. Compiling Android targets may result in confusing ' |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 807 | 'errors.\n' % |
| 808 | (android_ndk_home_path, ndk_version, _SUPPORTED_ANDROID_NDK_VERSIONS)) |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 809 | |
| 810 | # Now grab the NDK API level to use. Note that this is different from the |
| 811 | # SDK API level, as the NDK API level is effectively the *min* target SDK |
| 812 | # version. |
| 813 | platforms = os.path.join(android_ndk_home_path, 'platforms') |
| 814 | api_levels = sorted(os.listdir(platforms)) |
| 815 | api_levels = [ |
| 816 | x.replace('android-', '') for x in api_levels if 'android-' in x |
| 817 | ] |
| 818 | |
| 819 | def valid_api_level(api_level): |
| 820 | return os.path.exists( |
| 821 | os.path.join(android_ndk_home_path, 'platforms', |
| 822 | 'android-' + api_level)) |
| 823 | |
| 824 | android_ndk_api_level = prompt_loop_or_load_from_env( |
| 825 | environ_cp, |
| 826 | var_name='ANDROID_NDK_API_LEVEL', |
Jared Duke | a1309f4 | 2019-11-15 17:09:35 -0800 | [diff] [blame] | 827 | var_default='21', # 21 is required for ARM64 support. |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 828 | ask_for_var=('Please specify the (min) Android NDK API level to use. ' |
| 829 | '[Available levels: %s]') % api_levels, |
| 830 | check_success=valid_api_level, |
| 831 | error_msg='Android-%s is not present in the NDK path.') |
| 832 | |
| 833 | return android_ndk_api_level |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 834 | |
| 835 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 836 | def set_gcc_host_compiler_path(environ_cp): |
| 837 | """Set GCC_HOST_COMPILER_PATH.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 838 | default_gcc_host_compiler_path = which('gcc') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 839 | cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH') |
| 840 | |
| 841 | if os.path.islink(cuda_bin_symlink): |
| 842 | # os.readlink is only available in linux |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 843 | default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 844 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 845 | gcc_host_compiler_path = prompt_loop_or_load_from_env( |
| 846 | environ_cp, |
| 847 | var_name='GCC_HOST_COMPILER_PATH', |
| 848 | var_default=default_gcc_host_compiler_path, |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 849 | ask_for_var='Please specify which gcc should be used by nvcc as the host compiler.', |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 850 | check_success=os.path.exists, |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 851 | resolve_symlinks=True, |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 852 | error_msg='Invalid gcc path. %s cannot be found.', |
| 853 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 854 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 855 | write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path) |
| 856 | |
| 857 | |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 858 | def reformat_version_sequence(version_str, sequence_count): |
| 859 | """Reformat the version string to have the given number of sequences. |
| 860 | |
| 861 | For example: |
| 862 | Given (7, 2) -> 7.0 |
| 863 | (7.0.1, 2) -> 7.0 |
| 864 | (5, 1) -> 5 |
| 865 | (5.0.3.2, 1) -> 5 |
| 866 | |
| 867 | Args: |
| 868 | version_str: String, the version string. |
| 869 | sequence_count: int, an integer. |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 870 | |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 871 | Returns: |
| 872 | string, reformatted version string. |
| 873 | """ |
| 874 | v = version_str.split('.') |
| 875 | if len(v) < sequence_count: |
| 876 | v = v + (['0'] * (sequence_count - len(v))) |
| 877 | |
| 878 | return '.'.join(v[:sequence_count]) |
| 879 | |
| 880 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 881 | def set_tf_cuda_paths(environ_cp): |
| 882 | """Set TF_CUDA_PATHS.""" |
| 883 | ask_cuda_paths = ( |
| 884 | 'Please specify the comma-separated list of base paths to look for CUDA ' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 885 | 'libraries and headers. [Leave empty to use the default]: ') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 886 | tf_cuda_paths = get_from_env_or_user_or_default(environ_cp, 'TF_CUDA_PATHS', |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 887 | ask_cuda_paths, '') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 888 | if tf_cuda_paths: |
| 889 | environ_cp['TF_CUDA_PATHS'] = tf_cuda_paths |
| 890 | |
| 891 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 892 | def set_tf_cuda_version(environ_cp): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 893 | """Set TF_CUDA_VERSION.""" |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 894 | ask_cuda_version = ( |
| 895 | 'Please specify the CUDA SDK version you want to use. ' |
| 896 | '[Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 897 | tf_cuda_version = get_from_env_or_user_or_default(environ_cp, |
| 898 | 'TF_CUDA_VERSION', |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 899 | ask_cuda_version, |
| 900 | _DEFAULT_CUDA_VERSION) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 901 | environ_cp['TF_CUDA_VERSION'] = tf_cuda_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 902 | |
| 903 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 904 | def set_tf_cudnn_version(environ_cp): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 905 | """Set TF_CUDNN_VERSION.""" |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 906 | ask_cudnn_version = ( |
| 907 | 'Please specify the cuDNN version you want to use. ' |
| 908 | '[Leave empty to default to cuDNN %s]: ') % _DEFAULT_CUDNN_VERSION |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 909 | tf_cudnn_version = get_from_env_or_user_or_default(environ_cp, |
| 910 | 'TF_CUDNN_VERSION', |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 911 | ask_cudnn_version, |
| 912 | _DEFAULT_CUDNN_VERSION) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 913 | environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 914 | |
| 915 | |
Mingxing Tan | 1e7b0e4 | 2018-06-28 19:13:20 -0700 | [diff] [blame] | 916 | def is_cuda_compatible(lib, cuda_ver, cudnn_ver): |
| 917 | """Check compatibility between given library and cudnn/cudart libraries.""" |
| 918 | ldd_bin = which('ldd') or '/usr/bin/ldd' |
| 919 | ldd_out = run_shell([ldd_bin, lib], True) |
| 920 | ldd_out = ldd_out.split(os.linesep) |
| 921 | cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$') |
| 922 | cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$') |
| 923 | cudnn = None |
| 924 | cudart = None |
| 925 | cudnn_ok = True # assume no cudnn dependency by default |
| 926 | cuda_ok = True # assume no cuda dependency by default |
| 927 | for line in ldd_out: |
| 928 | if 'libcudnn.so' in line: |
| 929 | cudnn = cudnn_pattern.search(line) |
| 930 | cudnn_ok = False |
| 931 | elif 'libcudart.so' in line: |
| 932 | cudart = cuda_pattern.search(line) |
| 933 | cuda_ok = False |
| 934 | if cudnn and len(cudnn.group(1)): |
| 935 | cudnn = convert_version_to_int(cudnn.group(1)) |
| 936 | if cudart and len(cudart.group(1)): |
| 937 | cudart = convert_version_to_int(cudart.group(1)) |
| 938 | if cudnn is not None: |
| 939 | cudnn_ok = (cudnn == cudnn_ver) |
| 940 | if cudart is not None: |
| 941 | cuda_ok = (cudart == cuda_ver) |
| 942 | return cudnn_ok and cuda_ok |
| 943 | |
| 944 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 945 | def set_tf_tensorrt_version(environ_cp): |
| 946 | """Set TF_TENSORRT_VERSION.""" |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 947 | if not is_linux(): |
| 948 | raise ValueError('Currently TensorRT is only supported on Linux platform.') |
| 949 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 950 | if not int(environ_cp.get('TF_NEED_TENSORRT', False)): |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 951 | return |
| 952 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 953 | ask_tensorrt_version = ( |
| 954 | 'Please specify the TensorRT version you want to use. ' |
Guangda Lai | faa93ac | 2019-10-23 15:07:12 -0700 | [diff] [blame] | 955 | '[Leave empty to default to TensorRT %s]: ') % _DEFAULT_TENSORRT_VERSION |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 956 | tf_tensorrt_version = get_from_env_or_user_or_default( |
| 957 | environ_cp, 'TF_TENSORRT_VERSION', ask_tensorrt_version, |
| 958 | _DEFAULT_TENSORRT_VERSION) |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 959 | environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 960 | |
| 961 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 962 | def set_tf_nccl_version(environ_cp): |
| 963 | """Set TF_NCCL_VERSION.""" |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 964 | if not is_linux(): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 965 | raise ValueError('Currently NCCL is only supported on Linux platform.') |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 966 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 967 | if 'TF_NCCL_VERSION' in environ_cp: |
| 968 | return |
| 969 | |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 970 | ask_nccl_version = ( |
A. Unique TensorFlower | 53faa31 | 2018-10-05 08:46:54 -0700 | [diff] [blame] | 971 | 'Please specify the locally installed NCCL version you want to use. ' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 972 | '[Leave empty to use http://github.com/nvidia/nccl]: ') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 973 | tf_nccl_version = get_from_env_or_user_or_default(environ_cp, |
| 974 | 'TF_NCCL_VERSION', |
| 975 | ask_nccl_version, '') |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 976 | environ_cp['TF_NCCL_VERSION'] = tf_nccl_version |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 977 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 978 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 979 | def get_native_cuda_compute_capabilities(environ_cp): |
| 980 | """Get native cuda compute capabilities. |
| 981 | |
| 982 | Args: |
| 983 | environ_cp: copy of the os.environ. |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 984 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 985 | Returns: |
| 986 | string of native cuda compute capabilities, separated by comma. |
| 987 | """ |
| 988 | device_query_bin = os.path.join( |
| 989 | environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery') |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 990 | if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK): |
| 991 | try: |
| 992 | output = run_shell(device_query_bin).split('\n') |
| 993 | pattern = re.compile('[0-9]*\\.[0-9]*') |
| 994 | output = [pattern.search(x) for x in output if 'Capability' in x] |
| 995 | output = ','.join(x.group() for x in output if x is not None) |
| 996 | except subprocess.CalledProcessError: |
| 997 | output = '' |
| 998 | else: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 999 | output = '' |
| 1000 | return output |
| 1001 | |
| 1002 | |
| 1003 | def set_tf_cuda_compute_capabilities(environ_cp): |
| 1004 | """Set TF_CUDA_COMPUTE_CAPABILITIES.""" |
| 1005 | while True: |
| 1006 | native_cuda_compute_capabilities = get_native_cuda_compute_capabilities( |
| 1007 | environ_cp) |
| 1008 | if not native_cuda_compute_capabilities: |
| 1009 | default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES |
| 1010 | else: |
| 1011 | default_cuda_compute_capabilities = native_cuda_compute_capabilities |
| 1012 | |
| 1013 | ask_cuda_compute_capabilities = ( |
| 1014 | 'Please specify a list of comma-separated ' |
P Sudeepam | 5209356 | 2019-02-17 17:34:01 +0530 | [diff] [blame] | 1015 | 'CUDA compute capabilities you want to ' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1016 | 'build with.\nYou can find the compute ' |
| 1017 | 'capability of your device at: ' |
| 1018 | 'https://developer.nvidia.com/cuda-gpus.\nPlease' |
| 1019 | ' note that each additional compute ' |
| 1020 | 'capability significantly increases your ' |
P Sudeepam | 5209356 | 2019-02-17 17:34:01 +0530 | [diff] [blame] | 1021 | 'build time and binary size, and that ' |
| 1022 | 'TensorFlow only supports compute ' |
P Sudeepam | 765ceda | 2019-02-17 17:39:08 +0530 | [diff] [blame] | 1023 | 'capabilities >= 3.5 [Default is: %s]: ' % |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1024 | default_cuda_compute_capabilities) |
| 1025 | tf_cuda_compute_capabilities = get_from_env_or_user_or_default( |
| 1026 | environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES', |
| 1027 | ask_cuda_compute_capabilities, default_cuda_compute_capabilities) |
| 1028 | # Check whether all capabilities from the input is valid |
| 1029 | all_valid = True |
Maciej | d0f5bc1 | 2018-04-30 22:30:58 -0500 | [diff] [blame] | 1030 | # Remove all whitespace characters before splitting the string |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 1031 | # that users may insert by accident, as this will result in error |
Maciej | d0f5bc1 | 2018-04-30 22:30:58 -0500 | [diff] [blame] | 1032 | tf_cuda_compute_capabilities = ''.join(tf_cuda_compute_capabilities.split()) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1033 | for compute_capability in tf_cuda_compute_capabilities.split(','): |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1034 | m = re.match('[0-9]+.[0-9]+', compute_capability) |
| 1035 | if not m: |
Austin Anderson | 32202dc | 2019-02-19 10:46:27 -0800 | [diff] [blame] | 1036 | print('Invalid compute capability: %s' % compute_capability) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1037 | all_valid = False |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1038 | else: |
P Sudeepam | 5209356 | 2019-02-17 17:34:01 +0530 | [diff] [blame] | 1039 | ver = float(m.group(0)) |
A. Unique TensorFlower | 8dc2d0e | 2019-03-12 01:41:05 -0700 | [diff] [blame] | 1040 | if ver < 3.0: |
| 1041 | print('ERROR: TensorFlow only supports CUDA compute capabilities 3.0 ' |
Austin Anderson | 32202dc | 2019-02-19 10:46:27 -0800 | [diff] [blame] | 1042 | 'and higher. Please re-specify the list of compute ' |
| 1043 | 'capabilities excluding version %s.' % ver) |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1044 | all_valid = False |
A. Unique TensorFlower | 8dc2d0e | 2019-03-12 01:41:05 -0700 | [diff] [blame] | 1045 | if ver < 3.5: |
| 1046 | print('WARNING: XLA does not support CUDA compute capabilities ' |
| 1047 | 'lower than 3.5. Disable XLA when running on older GPUs.') |
P Sudeepam | 765ceda | 2019-02-17 17:39:08 +0530 | [diff] [blame] | 1048 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1049 | if all_valid: |
| 1050 | break |
| 1051 | |
| 1052 | # Reset and Retry |
| 1053 | environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = '' |
| 1054 | |
| 1055 | # Set TF_CUDA_COMPUTE_CAPABILITIES |
| 1056 | environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities |
| 1057 | write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES', |
| 1058 | tf_cuda_compute_capabilities) |
| 1059 | |
| 1060 | |
| 1061 | def set_other_cuda_vars(environ_cp): |
| 1062 | """Set other CUDA related variables.""" |
A. Unique TensorFlower | ab39198 | 2018-07-11 04:52:49 -0700 | [diff] [blame] | 1063 | # If CUDA is enabled, always use GPU during build and test. |
| 1064 | if environ_cp.get('TF_CUDA_CLANG') == '1': |
| 1065 | write_to_bazelrc('build --config=cuda_clang') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1066 | else: |
A. Unique TensorFlower | ab39198 | 2018-07-11 04:52:49 -0700 | [diff] [blame] | 1067 | write_to_bazelrc('build --config=cuda') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1068 | |
| 1069 | |
| 1070 | def set_host_cxx_compiler(environ_cp): |
| 1071 | """Set HOST_CXX_COMPILER.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1072 | default_cxx_host_compiler = which('g++') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1073 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1074 | host_cxx_compiler = prompt_loop_or_load_from_env( |
| 1075 | environ_cp, |
| 1076 | var_name='HOST_CXX_COMPILER', |
| 1077 | var_default=default_cxx_host_compiler, |
| 1078 | ask_for_var=('Please specify which C++ compiler should be used as the ' |
| 1079 | 'host C++ compiler.'), |
| 1080 | check_success=os.path.exists, |
| 1081 | error_msg='Invalid C++ compiler path. %s cannot be found.', |
| 1082 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1083 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1084 | write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler) |
| 1085 | |
| 1086 | |
| 1087 | def set_host_c_compiler(environ_cp): |
| 1088 | """Set HOST_C_COMPILER.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1089 | default_c_host_compiler = which('gcc') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1090 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1091 | host_c_compiler = prompt_loop_or_load_from_env( |
| 1092 | environ_cp, |
| 1093 | var_name='HOST_C_COMPILER', |
| 1094 | var_default=default_c_host_compiler, |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1095 | ask_for_var=('Please specify which C compiler should be used as the host ' |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1096 | 'C compiler.'), |
| 1097 | check_success=os.path.exists, |
| 1098 | error_msg='Invalid C compiler path. %s cannot be found.', |
| 1099 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1100 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1101 | write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler) |
| 1102 | |
| 1103 | |
| 1104 | def set_computecpp_toolkit_path(environ_cp): |
| 1105 | """Set COMPUTECPP_TOOLKIT_PATH.""" |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1106 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1107 | def toolkit_exists(toolkit_path): |
| 1108 | """Check if a computecpp toolkit path is valid.""" |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1109 | if is_linux(): |
| 1110 | sycl_rt_lib_path = 'lib/libComputeCpp.so' |
| 1111 | else: |
| 1112 | sycl_rt_lib_path = '' |
| 1113 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1114 | sycl_rt_lib_path_full = os.path.join(toolkit_path, sycl_rt_lib_path) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1115 | exists = os.path.exists(sycl_rt_lib_path_full) |
| 1116 | if not exists: |
| 1117 | print('Invalid SYCL %s library path. %s cannot be found' % |
| 1118 | (_TF_OPENCL_VERSION, sycl_rt_lib_path_full)) |
| 1119 | return exists |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1120 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1121 | computecpp_toolkit_path = prompt_loop_or_load_from_env( |
| 1122 | environ_cp, |
| 1123 | var_name='COMPUTECPP_TOOLKIT_PATH', |
| 1124 | var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH, |
| 1125 | ask_for_var=( |
| 1126 | 'Please specify the location where ComputeCpp for SYCL %s is ' |
| 1127 | 'installed.' % _TF_OPENCL_VERSION), |
| 1128 | check_success=toolkit_exists, |
| 1129 | error_msg='Invalid SYCL compiler path. %s cannot be found.', |
| 1130 | suppress_default_error=True) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1131 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1132 | write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH', |
| 1133 | computecpp_toolkit_path) |
| 1134 | |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1135 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1136 | def set_trisycl_include_dir(environ_cp): |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1137 | """Set TRISYCL_INCLUDE_DIR.""" |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 1138 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1139 | ask_trisycl_include_dir = ('Please specify the location of the triSYCL ' |
| 1140 | 'include directory. (Use --config=sycl_trisycl ' |
| 1141 | 'when building with Bazel) ' |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1142 | '[Default is %s]: ') % ( |
| 1143 | _DEFAULT_TRISYCL_INCLUDE_DIR) |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 1144 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1145 | while True: |
| 1146 | trisycl_include_dir = get_from_env_or_user_or_default( |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1147 | environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir, |
| 1148 | _DEFAULT_TRISYCL_INCLUDE_DIR) |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1149 | if os.path.exists(trisycl_include_dir): |
| 1150 | break |
| 1151 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1152 | print('Invalid triSYCL include directory, %s cannot be found' % |
| 1153 | (trisycl_include_dir)) |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1154 | |
| 1155 | # Set TRISYCL_INCLUDE_DIR |
| 1156 | environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1157 | write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', trisycl_include_dir) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1158 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1159 | |
William D. Irons | 391b21e | 2020-02-11 22:29:20 +0000 | [diff] [blame] | 1160 | def system_specific_test_config(environ_cp): |
A. Unique TensorFlower | 7bd8637 | 2019-03-21 15:19:30 -0700 | [diff] [blame] | 1161 | """Add default build and test flags required for TF tests to bazelrc.""" |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1162 | write_to_bazelrc('test --flaky_test_attempts=3') |
| 1163 | write_to_bazelrc('test --test_size_filters=small,medium') |
frreiss | 8b013a7 | 2019-11-20 11:36:27 -0800 | [diff] [blame] | 1164 | |
| 1165 | # Each instance of --test_tag_filters or --build_tag_filters overrides all |
| 1166 | # previous instances, so we need to build up a complete list and write a |
| 1167 | # single list of filters for the .bazelrc file. |
| 1168 | |
| 1169 | # Filters to use with both --test_tag_filters and --build_tag_filters |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1170 | test_and_build_filters = ['-benchmark-test', '-no_oss'] |
| 1171 | # Additional filters for --test_tag_filters beyond those in |
| 1172 | # test_and_build_filters |
| 1173 | test_only_filters = ['-oss_serial'] |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1174 | if is_windows(): |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1175 | test_and_build_filters.append('-no_windows') |
Deven Desai | f0bf514 | 2020-04-17 15:00:56 +0000 | [diff] [blame] | 1176 | if ((environ_cp.get('TF_NEED_CUDA', None) == '1') or |
| 1177 | (environ_cp.get('TF_NEED_ROCM', None) == '1')): |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1178 | test_and_build_filters += ['-no_windows_gpu', '-no_gpu'] |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1179 | else: |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1180 | test_and_build_filters.append('-gpu') |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1181 | elif is_macos(): |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1182 | test_and_build_filters += ['-gpu', '-nomac', '-no_mac'] |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1183 | elif is_linux(): |
Deven Desai | f0bf514 | 2020-04-17 15:00:56 +0000 | [diff] [blame] | 1184 | if ((environ_cp.get('TF_NEED_CUDA', None) == '1') or |
| 1185 | (environ_cp.get('TF_NEED_ROCM', None) == '1')): |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1186 | test_and_build_filters.append('-no_gpu') |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1187 | write_to_bazelrc('test --test_env=LD_LIBRARY_PATH') |
| 1188 | else: |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1189 | test_and_build_filters.append('-gpu') |
frreiss | 3adf417 | 2019-12-06 14:16:49 -0800 | [diff] [blame] | 1190 | |
| 1191 | # Disable tests with "v1only" tag in "v2" Bazel config, but not in "v1" config |
| 1192 | write_to_bazelrc('test:v1 --test_tag_filters=%s' % |
TensorFlower Gardener | e515908 | 2019-11-22 11:47:38 -0800 | [diff] [blame] | 1193 | ','.join(test_and_build_filters + test_only_filters)) |
frreiss | 3adf417 | 2019-12-06 14:16:49 -0800 | [diff] [blame] | 1194 | write_to_bazelrc('test:v1 --build_tag_filters=%s' % |
TensorFlower Gardener | e515908 | 2019-11-22 11:47:38 -0800 | [diff] [blame] | 1195 | ','.join(test_and_build_filters)) |
TensorFlower Gardener | 26a1de3 | 2019-12-17 07:11:48 -0800 | [diff] [blame] | 1196 | write_to_bazelrc( |
| 1197 | 'test:v2 --test_tag_filters=%s' % |
| 1198 | ','.join(test_and_build_filters + test_only_filters + ['-v1only'])) |
frreiss | 3adf417 | 2019-12-06 14:16:49 -0800 | [diff] [blame] | 1199 | write_to_bazelrc('test:v2 --build_tag_filters=%s' % |
TensorFlower Gardener | 26a1de3 | 2019-12-17 07:11:48 -0800 | [diff] [blame] | 1200 | ','.join(test_and_build_filters + ['-v1only'])) |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1201 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1202 | |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1203 | def set_system_libs_flag(environ_cp): |
| 1204 | syslibs = environ_cp.get('TF_SYSTEM_LIBS', '') |
TensorFlower Gardener | 61a8720 | 2018-10-01 12:25:39 -0700 | [diff] [blame] | 1205 | if syslibs: |
Jason Zaman | 5fc39bd | 2018-09-16 01:38:55 +0800 | [diff] [blame] | 1206 | if ',' in syslibs: |
| 1207 | syslibs = ','.join(sorted(syslibs.split(','))) |
| 1208 | else: |
| 1209 | syslibs = ','.join(sorted(syslibs.split())) |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1210 | write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs) |
| 1211 | |
Jason Zaman | 5fc39bd | 2018-09-16 01:38:55 +0800 | [diff] [blame] | 1212 | if 'PREFIX' in environ_cp: |
| 1213 | write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX']) |
| 1214 | if 'LIBDIR' in environ_cp: |
| 1215 | write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR']) |
| 1216 | if 'INCLUDEDIR' in environ_cp: |
| 1217 | write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR']) |
| 1218 | |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1219 | |
Austin Anderson | 6d7c376 | 2019-11-19 17:04:19 -0800 | [diff] [blame] | 1220 | def is_reduced_optimize_huge_functions_available(environ_cp): |
| 1221 | """Check to see if the system supports /d2ReducedOptimizeHugeFunctions. |
| 1222 | |
| 1223 | The above compiler flag is a new compiler flag introduced to the Visual Studio |
| 1224 | compiler in version 16.4 (available in Visual Studio 2019, Preview edition |
| 1225 | only, as of 2019-11-19). TensorFlow needs this flag to massively reduce |
| 1226 | compile times, but until 16.4 is officially released, we can't depend on it. |
| 1227 | |
Ahti Kitsik | 1dc9a3c | 2020-01-22 11:32:48 +0100 | [diff] [blame] | 1228 | See also https://groups.google.com/a/tensorflow.org/d/topic/build/SsW98Eo7l3o/discussion |
Austin Anderson | 6d7c376 | 2019-11-19 17:04:19 -0800 | [diff] [blame] | 1229 | |
| 1230 | Because it's very annoying to check this manually (to check the MSVC installed |
| 1231 | versions, you need to use the registry, and it's not clear if Bazel will be |
| 1232 | using that install version anyway), we expect enviroments who know they may |
| 1233 | use this flag to export TF_VC_VERSION=16.4 |
| 1234 | |
| 1235 | TODO(angerson, gunan): Remove this function when TensorFlow's minimum VS |
| 1236 | version is upgraded to 16.4. |
| 1237 | |
| 1238 | Arguments: |
| 1239 | environ_cp: Environment of the current execution |
| 1240 | |
| 1241 | Returns: |
| 1242 | boolean, whether or not /d2ReducedOptimizeHugeFunctions is available on this |
| 1243 | machine. |
| 1244 | """ |
| 1245 | return float(environ_cp.get('TF_VC_VERSION', '0')) >= 16.4 |
| 1246 | |
| 1247 | |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1248 | def set_windows_build_flags(environ_cp): |
| 1249 | """Set Windows specific build options.""" |
Austin Anderson | 6d7c376 | 2019-11-19 17:04:19 -0800 | [diff] [blame] | 1250 | if is_reduced_optimize_huge_functions_available(environ_cp): |
| 1251 | write_to_bazelrc( |
| 1252 | 'build --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions' |
| 1253 | ) |
| 1254 | |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1255 | if get_var( |
| 1256 | environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline', |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1257 | True, ('Would you like to override eigen strong inline for some C++ ' |
| 1258 | 'compilation to reduce the compilation time?'), |
| 1259 | 'Eigen strong inline overridden.', 'Not overriding eigen strong inline, ' |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1260 | 'some compilations could take more than 20 mins.'): |
| 1261 | # Due to a known MSVC compiler issue |
| 1262 | # https://github.com/tensorflow/tensorflow/issues/10521 |
| 1263 | # Overriding eigen strong inline speeds up the compiling of |
| 1264 | # conv_grad_ops_3d.cc and conv_ops_3d.cc by 20 minutes, |
| 1265 | # but this also hurts the performance. Let users decide what they want. |
| 1266 | write_to_bazelrc('build --define=override_eigen_strong_inline=true') |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1267 | |
A. Unique TensorFlower | 061c359 | 2017-11-13 14:21:04 -0800 | [diff] [blame] | 1268 | |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1269 | def config_info_line(name, help_text): |
| 1270 | """Helper function to print formatted help text for Bazel config options.""" |
| 1271 | print('\t--config=%-12s\t# %s' % (name, help_text)) |
| 1272 | |
| 1273 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1274 | def configure_ios(): |
| 1275 | """Configures TensorFlow for iOS builds. |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1276 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1277 | This function will only be executed if `is_macos()` is true. |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1278 | """ |
| 1279 | if not is_macos(): |
| 1280 | return |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1281 | for filepath in APPLE_BAZEL_FILES: |
| 1282 | existing_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath + '.apple') |
| 1283 | renamed_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath) |
| 1284 | symlink_force(existing_filepath, renamed_filepath) |
| 1285 | for filepath in IOS_FILES: |
| 1286 | filename = os.path.basename(filepath) |
| 1287 | new_filepath = os.path.join(_TF_WORKSPACE_ROOT, filename) |
| 1288 | symlink_force(filepath, new_filepath) |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1289 | |
| 1290 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1291 | def validate_cuda_config(environ_cp): |
| 1292 | """Run find_cuda_config.py and return cuda_toolkit_path, or None.""" |
| 1293 | |
| 1294 | def maybe_encode_env(env): |
| 1295 | """Encodes unicode in env to str on Windows python 2.x.""" |
| 1296 | if not is_windows() or sys.version_info[0] != 2: |
| 1297 | return env |
| 1298 | for k, v in env.items(): |
| 1299 | if isinstance(k, unicode): |
| 1300 | k = k.encode('ascii') |
| 1301 | if isinstance(v, unicode): |
| 1302 | v = v.encode('ascii') |
| 1303 | env[k] = v |
| 1304 | return env |
| 1305 | |
| 1306 | cuda_libraries = ['cuda', 'cudnn'] |
| 1307 | if is_linux(): |
A. Unique TensorFlower | 28fc9cc | 2019-05-01 14:17:54 -0700 | [diff] [blame] | 1308 | if int(environ_cp.get('TF_NEED_TENSORRT', False)): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1309 | cuda_libraries.append('tensorrt') |
A. Unique TensorFlower | b2e7f67 | 2019-04-30 09:20:36 -0700 | [diff] [blame] | 1310 | if environ_cp.get('TF_NCCL_VERSION', None): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1311 | cuda_libraries.append('nccl') |
| 1312 | |
| 1313 | proc = subprocess.Popen( |
| 1314 | [environ_cp['PYTHON_BIN_PATH'], 'third_party/gpus/find_cuda_config.py'] + |
| 1315 | cuda_libraries, |
| 1316 | stdout=subprocess.PIPE, |
| 1317 | env=maybe_encode_env(environ_cp)) |
| 1318 | |
| 1319 | if proc.wait(): |
| 1320 | # Errors from find_cuda_config.py were sent to stderr. |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 1321 | print('Asking for detailed CUDA configuration...\n') |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1322 | return False |
| 1323 | |
| 1324 | config = dict( |
| 1325 | tuple(line.decode('ascii').rstrip().split(': ')) for line in proc.stdout) |
| 1326 | |
| 1327 | print('Found CUDA %s in:' % config['cuda_version']) |
| 1328 | print(' %s' % config['cuda_library_dir']) |
| 1329 | print(' %s' % config['cuda_include_dir']) |
| 1330 | |
| 1331 | print('Found cuDNN %s in:' % config['cudnn_version']) |
| 1332 | print(' %s' % config['cudnn_library_dir']) |
| 1333 | print(' %s' % config['cudnn_include_dir']) |
| 1334 | |
| 1335 | if 'tensorrt_version' in config: |
| 1336 | print('Found TensorRT %s in:' % config['tensorrt_version']) |
| 1337 | print(' %s' % config['tensorrt_library_dir']) |
| 1338 | print(' %s' % config['tensorrt_include_dir']) |
| 1339 | |
| 1340 | if config.get('nccl_version', None): |
| 1341 | print('Found NCCL %s in:' % config['nccl_version']) |
| 1342 | print(' %s' % config['nccl_library_dir']) |
| 1343 | print(' %s' % config['nccl_include_dir']) |
| 1344 | |
| 1345 | print('\n') |
| 1346 | |
| 1347 | environ_cp['CUDA_TOOLKIT_PATH'] = config['cuda_toolkit_path'] |
| 1348 | return True |
| 1349 | |
| 1350 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1351 | def main(): |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1352 | global _TF_WORKSPACE_ROOT |
| 1353 | global _TF_BAZELRC |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 1354 | global _TF_CURRENT_BAZEL_VERSION |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1355 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1356 | parser = argparse.ArgumentParser() |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1357 | parser.add_argument( |
| 1358 | '--workspace', |
| 1359 | type=str, |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1360 | default=os.path.abspath(os.path.dirname(__file__)), |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1361 | help='The absolute path to your active Bazel workspace.') |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1362 | args = parser.parse_args() |
| 1363 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1364 | _TF_WORKSPACE_ROOT = args.workspace |
| 1365 | _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME) |
| 1366 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1367 | # Make a copy of os.environ to be clear when functions and getting and setting |
| 1368 | # environment variables. |
| 1369 | environ_cp = dict(os.environ) |
| 1370 | |
Mark Daoust | 44000ad | 2019-06-18 09:26:26 -0700 | [diff] [blame] | 1371 | current_bazel_version = check_bazel_version(_TF_MIN_BAZEL_VERSION, |
| 1372 | _TF_MAX_BAZEL_VERSION) |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 1373 | _TF_CURRENT_BAZEL_VERSION = convert_version_to_int(current_bazel_version) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1374 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1375 | reset_tf_configure_bazelrc() |
Yun Peng | 03e63a2 | 2018-11-07 11:18:53 +0100 | [diff] [blame] | 1376 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1377 | cleanup_makefile() |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 1378 | setup_python(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1379 | |
| 1380 | if is_windows(): |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1381 | environ_cp['TF_NEED_OPENCL_SYCL'] = '0' |
| 1382 | environ_cp['TF_NEED_COMPUTECPP'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1383 | environ_cp['TF_NEED_OPENCL'] = '0' |
| 1384 | environ_cp['TF_CUDA_CLANG'] = '0' |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 1385 | environ_cp['TF_NEED_TENSORRT'] = '0' |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1386 | # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on |
| 1387 | # Windows. |
| 1388 | environ_cp['TF_DOWNLOAD_CLANG'] = '0' |
A. Unique TensorFlower | 6e97fb3 | 2018-07-16 14:07:29 -0700 | [diff] [blame] | 1389 | environ_cp['TF_NEED_MPI'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1390 | |
| 1391 | if is_macos(): |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 1392 | environ_cp['TF_NEED_TENSORRT'] = '0' |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1393 | else: |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1394 | environ_cp['TF_CONFIGURE_IOS'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1395 | |
Gunhan Gulsoy | e733b05 | 2020-02-28 13:26:50 -0800 | [diff] [blame] | 1396 | if environ_cp.get('TF_ENABLE_XLA', '1') == '1': |
Gunhan Gulsoy | f4edbd9 | 2020-02-27 16:07:32 -0800 | [diff] [blame] | 1397 | write_to_bazelrc('build --config=xla') |
| 1398 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1399 | set_action_env_var( |
| 1400 | environ_cp, |
| 1401 | 'TF_NEED_OPENCL_SYCL', |
| 1402 | 'OpenCL SYCL', |
| 1403 | False, |
| 1404 | bazel_config_name='sycl') |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1405 | if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1406 | set_host_cxx_compiler(environ_cp) |
| 1407 | set_host_c_compiler(environ_cp) |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1408 | set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True) |
| 1409 | if environ_cp.get('TF_NEED_COMPUTECPP') == '1': |
| 1410 | set_computecpp_toolkit_path(environ_cp) |
| 1411 | else: |
| 1412 | set_trisycl_include_dir(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1413 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1414 | set_action_env_var( |
| 1415 | environ_cp, 'TF_NEED_ROCM', 'ROCm', False, bazel_config_name='rocm') |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1416 | if (environ_cp.get('TF_NEED_ROCM') == '1' and |
TensorFlower Gardener | 62e6016 | 2018-09-27 10:22:55 -0700 | [diff] [blame] | 1417 | 'LD_LIBRARY_PATH' in environ_cp and |
| 1418 | environ_cp.get('LD_LIBRARY_PATH') != '1'): |
| 1419 | write_action_env_to_bazelrc('LD_LIBRARY_PATH', |
| 1420 | environ_cp.get('LD_LIBRARY_PATH')) |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1421 | |
Deven Desai | f0bf514 | 2020-04-17 15:00:56 +0000 | [diff] [blame] | 1422 | if (environ_cp.get('TF_NEED_ROCM') == '1' and environ_cp.get('ROCM_PATH')): |
TensorFlower Gardener | 05fdea6 | 2020-04-23 17:12:38 -0700 | [diff] [blame] | 1423 | write_action_env_to_bazelrc('ROCM_PATH', environ_cp.get('ROCM_PATH')) |
| 1424 | write_action_env_to_bazelrc('ROCM_ROOT', environ_cp.get('ROCM_PATH')) |
Deven Desai | f0bf514 | 2020-04-17 15:00:56 +0000 | [diff] [blame] | 1425 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1426 | environ_cp['TF_NEED_CUDA'] = str( |
| 1427 | int(get_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False))) |
A. Unique TensorFlower | 24cbb2a | 2017-09-08 07:45:44 -0700 | [diff] [blame] | 1428 | if (environ_cp.get('TF_NEED_CUDA') == '1' and |
| 1429 | 'TF_CUDA_CONFIG_REPO' not in environ_cp): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1430 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1431 | set_action_env_var( |
| 1432 | environ_cp, |
| 1433 | 'TF_NEED_TENSORRT', |
| 1434 | 'TensorRT', |
| 1435 | False, |
| 1436 | bazel_config_name='tensorrt') |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1437 | |
| 1438 | environ_save = dict(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1439 | for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1440 | |
| 1441 | if validate_cuda_config(environ_cp): |
| 1442 | cuda_env_names = [ |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1443 | 'TF_CUDA_VERSION', |
| 1444 | 'TF_CUBLAS_VERSION', |
| 1445 | 'TF_CUDNN_VERSION', |
| 1446 | 'TF_TENSORRT_VERSION', |
| 1447 | 'TF_NCCL_VERSION', |
| 1448 | 'TF_CUDA_PATHS', |
A. Unique TensorFlower | b2e7f67 | 2019-04-30 09:20:36 -0700 | [diff] [blame] | 1449 | # Items below are for backwards compatibility when not using |
| 1450 | # TF_CUDA_PATHS. |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1451 | 'CUDA_TOOLKIT_PATH', |
| 1452 | 'CUDNN_INSTALL_PATH', |
| 1453 | 'NCCL_INSTALL_PATH', |
| 1454 | 'NCCL_HDR_PATH', |
| 1455 | 'TENSORRT_INSTALL_PATH' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1456 | ] |
A. Unique TensorFlower | b2e7f67 | 2019-04-30 09:20:36 -0700 | [diff] [blame] | 1457 | # Note: set_action_env_var above already writes to bazelrc. |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1458 | for name in cuda_env_names: |
| 1459 | if name in environ_cp: |
| 1460 | write_action_env_to_bazelrc(name, environ_cp[name]) |
| 1461 | break |
| 1462 | |
| 1463 | # Restore settings changed below if CUDA config could not be validated. |
| 1464 | environ_cp = dict(environ_save) |
| 1465 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1466 | set_tf_cuda_version(environ_cp) |
| 1467 | set_tf_cudnn_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1468 | if is_linux(): |
| 1469 | set_tf_tensorrt_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1470 | set_tf_nccl_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1471 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1472 | set_tf_cuda_paths(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1473 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1474 | else: |
| 1475 | raise UserInputError( |
| 1476 | 'Invalid CUDA setting were provided %d ' |
| 1477 | 'times in a row. Assuming to be a scripting mistake.' % |
| 1478 | _DEFAULT_PROMPT_ASK_ATTEMPTS) |
| 1479 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1480 | set_tf_cuda_compute_capabilities(environ_cp) |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 1481 | if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get( |
| 1482 | 'LD_LIBRARY_PATH') != '1': |
| 1483 | write_action_env_to_bazelrc('LD_LIBRARY_PATH', |
| 1484 | environ_cp.get('LD_LIBRARY_PATH')) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1485 | |
| 1486 | set_tf_cuda_clang(environ_cp) |
| 1487 | if environ_cp.get('TF_CUDA_CLANG') == '1': |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1488 | # Ask whether we should download the clang toolchain. |
| 1489 | set_tf_download_clang(environ_cp) |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 1490 | if environ_cp.get('TF_DOWNLOAD_CLANG') != '1': |
| 1491 | # Set up which clang we should use as the cuda / host compiler. |
| 1492 | set_clang_cuda_compiler_path(environ_cp) |
Ilya Biryukov | 1c3d02e | 2018-09-04 03:09:52 -0700 | [diff] [blame] | 1493 | else: |
| 1494 | # Use downloaded LLD for linking. |
| 1495 | write_to_bazelrc('build:cuda_clang --config=download_clang_use_lld') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1496 | else: |
| 1497 | # Set up which gcc nvcc should use as the host compiler |
| 1498 | # No need to set this on Windows |
| 1499 | if not is_windows(): |
| 1500 | set_gcc_host_compiler_path(environ_cp) |
| 1501 | set_other_cuda_vars(environ_cp) |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1502 | else: |
| 1503 | # CUDA not required. Ask whether we should download the clang toolchain and |
| 1504 | # use it for the CPU build. |
| 1505 | set_tf_download_clang(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1506 | |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1507 | # SYCL / ROCm / CUDA are mutually exclusive. |
| 1508 | # At most 1 GPU platform can be configured. |
| 1509 | gpu_platform_count = 0 |
| 1510 | if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': |
| 1511 | gpu_platform_count += 1 |
| 1512 | if environ_cp.get('TF_NEED_ROCM') == '1': |
| 1513 | gpu_platform_count += 1 |
| 1514 | if environ_cp.get('TF_NEED_CUDA') == '1': |
| 1515 | gpu_platform_count += 1 |
| 1516 | if gpu_platform_count >= 2: |
| 1517 | raise UserInputError('SYCL / CUDA / ROCm are mututally exclusive. ' |
| 1518 | 'At most 1 GPU platform can be configured.') |
| 1519 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1520 | set_cc_opt_flags(environ_cp) |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1521 | set_system_libs_flag(environ_cp) |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1522 | if is_windows(): |
| 1523 | set_windows_build_flags(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1524 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1525 | if get_var(environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', False, |
| 1526 | ('Would you like to interactively configure ./WORKSPACE for ' |
| 1527 | 'Android builds?'), 'Searching for NDK and SDK installations.', |
| 1528 | 'Not configuring the WORKSPACE for Android builds.'): |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 1529 | create_android_ndk_rule(environ_cp) |
| 1530 | create_android_sdk_rule(environ_cp) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1531 | |
William D. Irons | 391b21e | 2020-02-11 22:29:20 +0000 | [diff] [blame] | 1532 | system_specific_test_config(environ_cp) |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1533 | |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 1534 | set_action_env_var(environ_cp, 'TF_CONFIGURE_IOS', 'iOS', False) |
| 1535 | if environ_cp.get('TF_CONFIGURE_IOS') == '1': |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1536 | configure_ios() |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1537 | |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1538 | print('Preconfigured Bazel build configs. You can use any of the below by ' |
| 1539 | 'adding "--config=<>" to your build command. See .bazelrc for more ' |
| 1540 | 'details.') |
| 1541 | config_info_line('mkl', 'Build with MKL support.') |
| 1542 | config_info_line('monolithic', 'Config for mostly static monolithic build.') |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1543 | config_info_line('ngraph', 'Build with Intel nGraph support.') |
A. Unique TensorFlower | a6bf9c8 | 2019-02-26 10:08:35 -0800 | [diff] [blame] | 1544 | config_info_line('numa', 'Build with NUMA support.') |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 1545 | config_info_line( |
| 1546 | 'dynamic_kernels', |
| 1547 | '(Experimental) Build kernels into separate shared objects.') |
Anna R | aedf674 | 2019-06-12 11:30:51 -0700 | [diff] [blame] | 1548 | config_info_line('v2', 'Build TensorFlow 2.x instead of 1.x.') |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1549 | |
| 1550 | print('Preconfigured Bazel build configs to DISABLE default on features:') |
| 1551 | config_info_line('noaws', 'Disable AWS S3 filesystem support.') |
| 1552 | config_info_line('nogcp', 'Disable GCP support.') |
| 1553 | config_info_line('nohdfs', 'Disable HDFS support.') |
Gunhan Gulsoy | eea8168 | 2018-11-26 16:51:23 -0800 | [diff] [blame] | 1554 | config_info_line('nonccl', 'Disable NVIDIA NCCL support.') |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1555 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1556 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1557 | if __name__ == '__main__': |
| 1558 | main() |