A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | # ============================================================================== |
| 15 | """configure script to get build parameters from user.""" |
| 16 | |
| 17 | from __future__ import absolute_import |
| 18 | from __future__ import division |
| 19 | from __future__ import print_function |
| 20 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 21 | import argparse |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 22 | import errno |
Austin Anderson | b1f946a | 2021-01-26 11:07:14 -0800 | [diff] [blame] | 23 | import glob |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 24 | import os |
| 25 | import platform |
| 26 | import re |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 27 | import subprocess |
| 28 | import sys |
| 29 | |
Andrew Selle | c9885ea | 2017-11-06 09:37:03 -0800 | [diff] [blame] | 30 | # pylint: disable=g-import-not-at-top |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 31 | try: |
| 32 | from shutil import which |
| 33 | except ImportError: |
| 34 | from distutils.spawn import find_executable as which |
Andrew Selle | c9885ea | 2017-11-06 09:37:03 -0800 | [diff] [blame] | 35 | # pylint: enable=g-import-not-at-top |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 36 | |
Christian Sigg | 8dff6b9 | 2019-12-09 04:05:46 -0800 | [diff] [blame] | 37 | _DEFAULT_CUDA_VERSION = '10' |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 38 | _DEFAULT_CUDNN_VERSION = '7' |
Guangda Lai | faa93ac | 2019-10-23 15:07:12 -0700 | [diff] [blame] | 39 | _DEFAULT_TENSORRT_VERSION = '6' |
Smit Hinsu | fe7d1d9 | 2018-07-14 13:16:58 -0700 | [diff] [blame] | 40 | _DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 41 | |
Ian Hua | 6b51112 | 2021-08-11 21:38:50 +0100 | [diff] [blame] | 42 | _SUPPORTED_ANDROID_NDK_VERSIONS = [ |
| 43 | 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 |
| 44 | ] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 45 | |
| 46 | _DEFAULT_PROMPT_ASK_ATTEMPTS = 10 |
| 47 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 48 | _TF_BAZELRC_FILENAME = '.tf_configure.bazelrc' |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 49 | _TF_WORKSPACE_ROOT = '' |
| 50 | _TF_BAZELRC = '' |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 51 | _TF_CURRENT_BAZEL_VERSION = None |
Mihai Maruseac | 83ba6a3 | 2021-01-21 17:31:23 -0800 | [diff] [blame] | 52 | _TF_MIN_BAZEL_VERSION = '3.7.2' |
Ian Hua | 6b51112 | 2021-08-11 21:38:50 +0100 | [diff] [blame] | 53 | _TF_MAX_BAZEL_VERSION = '4.99.0' |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 54 | |
Jason Furmanek | 7c23415 | 2018-09-26 04:44:12 +0000 | [diff] [blame] | 55 | NCCL_LIB_PATHS = [ |
TensorFlower Gardener | ea5c529 | 2018-10-01 17:28:26 -0700 | [diff] [blame] | 56 | 'lib64/', 'lib/powerpc64le-linux-gnu/', 'lib/x86_64-linux-gnu/', '' |
Jason Furmanek | 7c23415 | 2018-09-26 04:44:12 +0000 | [diff] [blame] | 57 | ] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 58 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 59 | # List of files to configure when building Bazel on Apple platforms. |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 60 | APPLE_BAZEL_FILES = [ |
TensorFlower Gardener | 5b77c61 | 2020-12-22 06:19:25 -0800 | [diff] [blame] | 61 | 'tensorflow/lite/ios/BUILD', 'tensorflow/lite/objc/BUILD', |
YoungSeok Yoon | 5a063c2 | 2020-11-23 17:14:56 -0800 | [diff] [blame] | 62 | 'tensorflow/lite/swift/BUILD', |
YoungSeok Yoon | 7089aaa | 2019-11-11 21:36:17 -0800 | [diff] [blame] | 63 | 'tensorflow/lite/tools/benchmark/experimental/ios/BUILD' |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 64 | ] |
| 65 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 66 | # List of files to move when building for iOS. |
| 67 | IOS_FILES = [ |
YoungSeok Yoon | 5a063c2 | 2020-11-23 17:14:56 -0800 | [diff] [blame] | 68 | 'tensorflow/lite/objc/TensorFlowLiteObjC.podspec', |
| 69 | 'tensorflow/lite/swift/TensorFlowLiteSwift.podspec', |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 70 | ] |
| 71 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 72 | |
| 73 | class UserInputError(Exception): |
| 74 | pass |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 75 | |
| 76 | |
| 77 | def is_windows(): |
| 78 | return platform.system() == 'Windows' |
| 79 | |
| 80 | |
| 81 | def is_linux(): |
| 82 | return platform.system() == 'Linux' |
| 83 | |
| 84 | |
| 85 | def is_macos(): |
| 86 | return platform.system() == 'Darwin' |
| 87 | |
| 88 | |
| 89 | def is_ppc64le(): |
| 90 | return platform.machine() == 'ppc64le' |
| 91 | |
| 92 | |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 93 | def is_cygwin(): |
| 94 | return platform.system().startswith('CYGWIN_NT') |
| 95 | |
| 96 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 97 | def get_input(question): |
| 98 | try: |
| 99 | try: |
| 100 | answer = raw_input(question) |
| 101 | except NameError: |
| 102 | answer = input(question) # pylint: disable=bad-builtin |
| 103 | except EOFError: |
| 104 | answer = '' |
| 105 | return answer |
| 106 | |
| 107 | |
| 108 | def symlink_force(target, link_name): |
| 109 | """Force symlink, equivalent of 'ln -sf'. |
| 110 | |
| 111 | Args: |
| 112 | target: items to link to. |
| 113 | link_name: name of the link. |
| 114 | """ |
| 115 | try: |
| 116 | os.symlink(target, link_name) |
| 117 | except OSError as e: |
| 118 | if e.errno == errno.EEXIST: |
| 119 | os.remove(link_name) |
| 120 | os.symlink(target, link_name) |
| 121 | else: |
| 122 | raise e |
| 123 | |
| 124 | |
| 125 | def sed_in_place(filename, old, new): |
| 126 | """Replace old string with new string in file. |
| 127 | |
| 128 | Args: |
| 129 | filename: string for filename. |
| 130 | old: string to replace. |
| 131 | new: new string to replace to. |
| 132 | """ |
| 133 | with open(filename, 'r') as f: |
| 134 | filedata = f.read() |
| 135 | newdata = filedata.replace(old, new) |
| 136 | with open(filename, 'w') as f: |
| 137 | f.write(newdata) |
| 138 | |
| 139 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 140 | def write_to_bazelrc(line): |
| 141 | with open(_TF_BAZELRC, 'a') as f: |
| 142 | f.write(line + '\n') |
| 143 | |
| 144 | |
| 145 | def write_action_env_to_bazelrc(var_name, var): |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 146 | write_to_bazelrc('build --action_env {}="{}"'.format(var_name, str(var))) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 147 | |
| 148 | |
Wallyss Lima | 7237104 | 2019-10-03 12:55:30 -0300 | [diff] [blame] | 149 | def run_shell(cmd, allow_non_zero=False, stderr=None): |
| 150 | if stderr is None: |
| 151 | stderr = sys.stdout |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 152 | if allow_non_zero: |
| 153 | try: |
Wallyss Lima | 7237104 | 2019-10-03 12:55:30 -0300 | [diff] [blame] | 154 | output = subprocess.check_output(cmd, stderr=stderr) |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 155 | except subprocess.CalledProcessError as e: |
| 156 | output = e.output |
| 157 | else: |
Wallyss Lima | 7237104 | 2019-10-03 12:55:30 -0300 | [diff] [blame] | 158 | output = subprocess.check_output(cmd, stderr=stderr) |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 159 | return output.decode('UTF-8').strip() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 160 | |
| 161 | |
| 162 | def cygpath(path): |
| 163 | """Convert path from posix to windows.""" |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 164 | return os.path.abspath(path).replace('\\', '/') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 165 | |
| 166 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 167 | def get_python_path(environ_cp, python_bin_path): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 168 | """Get the python site package paths.""" |
| 169 | python_paths = [] |
| 170 | if environ_cp.get('PYTHONPATH'): |
| 171 | python_paths = environ_cp.get('PYTHONPATH').split(':') |
| 172 | try: |
Wallyss Lima | 484304e | 2019-10-03 12:58:20 -0300 | [diff] [blame] | 173 | stderr = open(os.devnull, 'wb') |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 174 | library_paths = run_shell([ |
| 175 | python_bin_path, '-c', |
| 176 | 'import site; print("\\n".join(site.getsitepackages()))' |
TensorFlower Gardener | 839a436 | 2019-12-30 02:23:29 -0800 | [diff] [blame] | 177 | ], |
| 178 | stderr=stderr).split('\n') |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 179 | except subprocess.CalledProcessError: |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 180 | library_paths = [ |
| 181 | run_shell([ |
| 182 | python_bin_path, '-c', |
| 183 | 'from distutils.sysconfig import get_python_lib;' |
| 184 | 'print(get_python_lib())' |
| 185 | ]) |
| 186 | ] |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 187 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 188 | all_paths = set(python_paths + library_paths) |
Reed Wanderman-Milne | 03bd8a3 | 2021-02-10 07:41:07 -0800 | [diff] [blame] | 189 | # Sort set so order is deterministic |
| 190 | all_paths = sorted(all_paths) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 191 | |
| 192 | paths = [] |
| 193 | for path in all_paths: |
| 194 | if os.path.isdir(path): |
| 195 | paths.append(path) |
| 196 | return paths |
| 197 | |
| 198 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 199 | def get_python_major_version(python_bin_path): |
| 200 | """Get the python major version.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 201 | return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])']) |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 202 | |
| 203 | |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 204 | def setup_python(environ_cp): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 205 | """Setup python related env variables.""" |
| 206 | # Get PYTHON_BIN_PATH, default is the current running python. |
| 207 | default_python_bin_path = sys.executable |
| 208 | ask_python_bin_path = ('Please specify the location of python. [Default is ' |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 209 | '{}]: ').format(default_python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 210 | while True: |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 211 | python_bin_path = get_from_env_or_user_or_default(environ_cp, |
| 212 | 'PYTHON_BIN_PATH', |
| 213 | ask_python_bin_path, |
| 214 | default_python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 215 | # Check if the path is valid |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 216 | if os.path.isfile(python_bin_path) and os.access(python_bin_path, os.X_OK): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 217 | break |
| 218 | elif not os.path.exists(python_bin_path): |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 219 | print('Invalid python path: {} cannot be found.'.format(python_bin_path)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 220 | else: |
TensorFlower Gardener | 9784cf2 | 2020-05-12 15:05:26 -0700 | [diff] [blame] | 221 | print('{} is not executable. Is it the python binary?'.format( |
| 222 | python_bin_path)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 223 | environ_cp['PYTHON_BIN_PATH'] = '' |
| 224 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 225 | # Convert python path to Windows style before checking lib and version |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 226 | if is_windows() or is_cygwin(): |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 227 | python_bin_path = cygpath(python_bin_path) |
| 228 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 229 | # Get PYTHON_LIB_PATH |
| 230 | python_lib_path = environ_cp.get('PYTHON_LIB_PATH') |
| 231 | if not python_lib_path: |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 232 | python_lib_paths = get_python_path(environ_cp, python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 233 | if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1': |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 234 | python_lib_path = python_lib_paths[0] |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 235 | else: |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 236 | print('Found possible Python library paths:\n %s' % |
| 237 | '\n '.join(python_lib_paths)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 238 | default_python_lib_path = python_lib_paths[0] |
| 239 | python_lib_path = get_input( |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 240 | 'Please input the desired Python library path to use. ' |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 241 | 'Default is [{}]\n'.format(python_lib_paths[0])) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 242 | if not python_lib_path: |
| 243 | python_lib_path = default_python_lib_path |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 244 | environ_cp['PYTHON_LIB_PATH'] = python_lib_path |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 245 | |
A. Unique TensorFlower | cfccbdb | 2019-06-24 11:01:58 -0700 | [diff] [blame] | 246 | python_major_version = get_python_major_version(python_bin_path) |
| 247 | if python_major_version == '2': |
| 248 | write_to_bazelrc('build --host_force_python=PY2') |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 249 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 250 | # Convert python path to Windows style before writing into bazel.rc |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 251 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 252 | python_lib_path = cygpath(python_lib_path) |
| 253 | |
| 254 | # Set-up env variables used by python_configure.bzl |
| 255 | write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path) |
| 256 | write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path) |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 257 | write_to_bazelrc('build --python_path=\"{}"'.format(python_bin_path)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 258 | environ_cp['PYTHON_BIN_PATH'] = python_bin_path |
| 259 | |
William D. Irons | dcc76a5 | 2018-11-20 10:35:18 -0600 | [diff] [blame] | 260 | # If choosen python_lib_path is from a path specified in the PYTHONPATH |
| 261 | # variable, need to tell bazel to include PYTHONPATH |
| 262 | if environ_cp.get('PYTHONPATH'): |
| 263 | python_paths = environ_cp.get('PYTHONPATH').split(':') |
| 264 | if python_lib_path in python_paths: |
TensorFlower Gardener | 968cd18 | 2018-11-28 11:33:16 -0800 | [diff] [blame] | 265 | write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH')) |
William D. Irons | dcc76a5 | 2018-11-20 10:35:18 -0600 | [diff] [blame] | 266 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 267 | # Write tools/python_bin_path.sh |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 268 | with open( |
| 269 | os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'), |
| 270 | 'w') as f: |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 271 | f.write('export PYTHON_BIN_PATH="{}"'.format(python_bin_path)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 272 | |
| 273 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 274 | def reset_tf_configure_bazelrc(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 275 | """Reset file that contains customized config settings.""" |
| 276 | open(_TF_BAZELRC, 'w').close() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 277 | |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 278 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 279 | def cleanup_makefile(): |
| 280 | """Delete any leftover BUILD files from the Makefile build. |
| 281 | |
| 282 | These files could interfere with Bazel parsing. |
| 283 | """ |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 284 | makefile_download_dir = os.path.join(_TF_WORKSPACE_ROOT, 'tensorflow', |
| 285 | 'contrib', 'makefile', 'downloads') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 286 | if os.path.isdir(makefile_download_dir): |
| 287 | for root, _, filenames in os.walk(makefile_download_dir): |
| 288 | for f in filenames: |
| 289 | if f.endswith('BUILD'): |
| 290 | os.remove(os.path.join(root, f)) |
| 291 | |
| 292 | |
| 293 | def get_var(environ_cp, |
| 294 | var_name, |
| 295 | query_item, |
| 296 | enabled_by_default, |
| 297 | question=None, |
| 298 | yes_reply=None, |
| 299 | no_reply=None): |
| 300 | """Get boolean input from user. |
| 301 | |
| 302 | If var_name is not set in env, ask user to enable query_item or not. If the |
| 303 | response is empty, use the default. |
| 304 | |
| 305 | Args: |
| 306 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 307 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
| 308 | query_item: string for feature related to the variable, e.g. "CUDA for |
| 309 | Nvidia GPUs". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 310 | enabled_by_default: boolean for default behavior. |
| 311 | question: optional string for how to ask for user input. |
Michael Case | d90054e | 2018-02-07 14:36:00 -0800 | [diff] [blame] | 312 | yes_reply: optional string for reply when feature is enabled. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 313 | no_reply: optional string for reply when feature is disabled. |
| 314 | |
| 315 | Returns: |
| 316 | boolean value of the variable. |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 317 | |
| 318 | Raises: |
| 319 | UserInputError: if an environment variable is set, but it cannot be |
| 320 | interpreted as a boolean indicator, assume that the user has made a |
| 321 | scripting error, and will continue to provide invalid input. |
| 322 | Raise the error to avoid infinitely looping. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 323 | """ |
| 324 | if not question: |
TensorFlower Gardener | 9784cf2 | 2020-05-12 15:05:26 -0700 | [diff] [blame] | 325 | question = 'Do you wish to build TensorFlow with {} support?'.format( |
| 326 | query_item) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 327 | if not yes_reply: |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 328 | yes_reply = '{} support will be enabled for TensorFlow.'.format(query_item) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 329 | if not no_reply: |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 330 | no_reply = 'No {}'.format(yes_reply) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 331 | |
| 332 | yes_reply += '\n' |
| 333 | no_reply += '\n' |
| 334 | |
| 335 | if enabled_by_default: |
| 336 | question += ' [Y/n]: ' |
| 337 | else: |
| 338 | question += ' [y/N]: ' |
| 339 | |
| 340 | var = environ_cp.get(var_name) |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 341 | if var is not None: |
| 342 | var_content = var.strip().lower() |
| 343 | true_strings = ('1', 't', 'true', 'y', 'yes') |
| 344 | false_strings = ('0', 'f', 'false', 'n', 'no') |
| 345 | if var_content in true_strings: |
| 346 | var = True |
| 347 | elif var_content in false_strings: |
| 348 | var = False |
| 349 | else: |
| 350 | raise UserInputError( |
| 351 | 'Environment variable %s must be set as a boolean indicator.\n' |
| 352 | 'The following are accepted as TRUE : %s.\n' |
| 353 | 'The following are accepted as FALSE: %s.\n' |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 354 | 'Current value is %s.' % |
| 355 | (var_name, ', '.join(true_strings), ', '.join(false_strings), var)) |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 356 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 357 | while var is None: |
| 358 | user_input_origin = get_input(question) |
| 359 | user_input = user_input_origin.strip().lower() |
| 360 | if user_input == 'y': |
| 361 | print(yes_reply) |
| 362 | var = True |
| 363 | elif user_input == 'n': |
| 364 | print(no_reply) |
| 365 | var = False |
| 366 | elif not user_input: |
| 367 | if enabled_by_default: |
| 368 | print(yes_reply) |
| 369 | var = True |
| 370 | else: |
| 371 | print(no_reply) |
| 372 | var = False |
| 373 | else: |
marload | c6d2369 | 2020-05-11 15:37:57 +0900 | [diff] [blame] | 374 | print('Invalid selection: {}'.format(user_input_origin)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 375 | return var |
| 376 | |
| 377 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 378 | def set_build_var(environ_cp, |
| 379 | var_name, |
| 380 | query_item, |
| 381 | option_name, |
| 382 | enabled_by_default, |
| 383 | bazel_config_name=None): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 384 | """Set if query_item will be enabled for the build. |
| 385 | |
| 386 | Ask user if query_item will be enabled. Default is used if no input is given. |
| 387 | Set subprocess environment variable and write to .bazelrc if enabled. |
| 388 | |
| 389 | Args: |
| 390 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 391 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
| 392 | query_item: string for feature related to the variable, e.g. "CUDA for |
| 393 | Nvidia GPUs". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 394 | option_name: string for option to define in .bazelrc. |
| 395 | enabled_by_default: boolean for default behavior. |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 396 | bazel_config_name: Name for Bazel --config argument to enable build feature. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 397 | """ |
| 398 | |
| 399 | var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default))) |
| 400 | environ_cp[var_name] = var |
| 401 | if var == '1': |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 402 | write_to_bazelrc('build:%s --define %s=true' % |
| 403 | (bazel_config_name, option_name)) |
Yifei Feng | ec451f5 | 2018-10-05 12:53:50 -0700 | [diff] [blame] | 404 | write_to_bazelrc('build --config=%s' % bazel_config_name) |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 405 | elif bazel_config_name is not None: |
| 406 | # TODO(mikecase): Migrate all users of configure.py to use --config Bazel |
| 407 | # options and not to set build configs through environment variables. |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 408 | write_to_bazelrc('build:%s --define %s=true' % |
| 409 | (bazel_config_name, option_name)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 410 | |
| 411 | |
| 412 | def set_action_env_var(environ_cp, |
| 413 | var_name, |
| 414 | query_item, |
| 415 | enabled_by_default, |
| 416 | question=None, |
| 417 | yes_reply=None, |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 418 | no_reply=None, |
| 419 | bazel_config_name=None): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 420 | """Set boolean action_env variable. |
| 421 | |
| 422 | Ask user if query_item will be enabled. Default is used if no input is given. |
| 423 | Set environment variable and write to .bazelrc. |
| 424 | |
| 425 | Args: |
| 426 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 427 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
| 428 | query_item: string for feature related to the variable, e.g. "CUDA for |
| 429 | Nvidia GPUs". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 430 | enabled_by_default: boolean for default behavior. |
| 431 | question: optional string for how to ask for user input. |
Michael Case | d90054e | 2018-02-07 14:36:00 -0800 | [diff] [blame] | 432 | yes_reply: optional string for reply when feature is enabled. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 433 | no_reply: optional string for reply when feature is disabled. |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 434 | bazel_config_name: adding config to .bazelrc instead of action_env. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 435 | """ |
| 436 | var = int( |
| 437 | get_var(environ_cp, var_name, query_item, enabled_by_default, question, |
| 438 | yes_reply, no_reply)) |
| 439 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 440 | if not bazel_config_name: |
| 441 | write_action_env_to_bazelrc(var_name, var) |
| 442 | elif var: |
| 443 | write_to_bazelrc('build --config=%s' % bazel_config_name) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 444 | environ_cp[var_name] = str(var) |
| 445 | |
| 446 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 447 | def convert_version_to_int(version): |
| 448 | """Convert a version number to a integer that can be used to compare. |
| 449 | |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 450 | Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The |
| 451 | 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored. |
| 452 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 453 | Args: |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 454 | version: a version to be converted |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 455 | |
| 456 | Returns: |
| 457 | An integer if converted successfully, otherwise return None. |
| 458 | """ |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 459 | version = version.split('-')[0] |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 460 | version_segments = version.split('.') |
Austin Anderson | 87ea41d | 2019-04-04 10:03:50 -0700 | [diff] [blame] | 461 | # Treat "0.24" as "0.24.0" |
| 462 | if len(version_segments) == 2: |
| 463 | version_segments.append('0') |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 464 | for seg in version_segments: |
| 465 | if not seg.isdigit(): |
| 466 | return None |
| 467 | |
| 468 | version_str = ''.join(['%03d' % int(seg) for seg in version_segments]) |
| 469 | return int(version_str) |
| 470 | |
| 471 | |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 472 | def check_bazel_version(min_version, max_version): |
| 473 | """Check installed bazel version is between min_version and max_version. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 474 | |
| 475 | Args: |
Mihai Maruseac | 4db860f | 2019-04-19 08:52:10 -0700 | [diff] [blame] | 476 | min_version: string for minimum bazel version (must exist!). |
| 477 | max_version: string for maximum bazel version (must exist!). |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 478 | |
| 479 | Returns: |
| 480 | The bazel version detected. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 481 | """ |
Ian Hua | 6b51112 | 2021-08-11 21:38:50 +0100 | [diff] [blame] | 482 | bazel_executable = which('bazel') |
| 483 | if bazel_executable is None: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 484 | print('Cannot find bazel. Please install bazel.') |
hedgehog91 | 668eaa2 | 2020-06-24 21:36:46 +0300 | [diff] [blame] | 485 | sys.exit(1) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 486 | |
A. Unique TensorFlower | 56a99d6 | 2020-05-08 09:28:28 -0700 | [diff] [blame] | 487 | stderr = open(os.devnull, 'wb') |
Ian Hua | 6b51112 | 2021-08-11 21:38:50 +0100 | [diff] [blame] | 488 | curr_version = run_shell([bazel_executable, '--version'], |
Amit Patankar | a5393e9 | 2020-05-28 14:29:56 -0700 | [diff] [blame] | 489 | allow_non_zero=True, |
| 490 | stderr=stderr) |
A. Unique TensorFlower | 56a99d6 | 2020-05-08 09:28:28 -0700 | [diff] [blame] | 491 | if curr_version.startswith('bazel '): |
| 492 | curr_version = curr_version.split('bazel ')[1] |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 493 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 494 | min_version_int = convert_version_to_int(min_version) |
| 495 | curr_version_int = convert_version_to_int(curr_version) |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 496 | max_version_int = convert_version_to_int(max_version) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 497 | |
| 498 | # Check if current bazel version can be detected properly. |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 499 | if not curr_version_int: |
| 500 | print('WARNING: current bazel installation is not a release version.') |
| 501 | print('Make sure you are running at least bazel %s' % min_version) |
| 502 | return curr_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 503 | |
Michael Case | d94271a | 2017-08-22 17:26:52 -0700 | [diff] [blame] | 504 | print('You have bazel %s installed.' % curr_version) |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 505 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 506 | if curr_version_int < min_version_int: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 507 | print('Please upgrade your bazel installation to version %s or higher to ' |
| 508 | 'build TensorFlow!' % min_version) |
Jason Zaman | b41761c | 2018-10-14 11:28:53 +0800 | [diff] [blame] | 509 | sys.exit(1) |
TensorFlower Gardener | 78c246b | 2018-12-13 12:37:42 -0800 | [diff] [blame] | 510 | if (curr_version_int > max_version_int and |
| 511 | 'TF_IGNORE_MAX_BAZEL_VERSION' not in os.environ): |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 512 | print('Please downgrade your bazel installation to version %s or lower to ' |
Mihai Maruseac | e0963c4 | 2018-12-20 14:27:40 -0800 | [diff] [blame] | 513 | 'build TensorFlow! To downgrade: download the installer for the old ' |
| 514 | 'version (from https://github.com/bazelbuild/bazel/releases) then ' |
| 515 | 'run the installer.' % max_version) |
Jason Zaman | b41761c | 2018-10-14 11:28:53 +0800 | [diff] [blame] | 516 | sys.exit(1) |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 517 | return curr_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 518 | |
| 519 | |
| 520 | def set_cc_opt_flags(environ_cp): |
| 521 | """Set up architecture-dependent optimization flags. |
| 522 | |
| 523 | Also append CC optimization flags to bazel.rc.. |
| 524 | |
| 525 | Args: |
| 526 | environ_cp: copy of the os.environ. |
| 527 | """ |
| 528 | if is_ppc64le(): |
| 529 | # gcc on ppc64le does not support -march, use mcpu instead |
| 530 | default_cc_opt_flags = '-mcpu=native' |
A. Unique TensorFlower | 1bba94a | 2018-04-04 15:45:20 -0700 | [diff] [blame] | 531 | elif is_windows(): |
| 532 | default_cc_opt_flags = '/arch:AVX' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 533 | else: |
Mihai Maruseac | e0a6a8c | 2021-01-05 16:29:48 -0800 | [diff] [blame] | 534 | # On all other platforms, no longer use `-march=native` as this can result |
| 535 | # in instructions that are too modern being generated. Users that want |
| 536 | # maximum performance should compile TF in their environment and can pass |
| 537 | # `-march=native` there. |
| 538 | # See https://github.com/tensorflow/tensorflow/issues/45744 and duplicates |
| 539 | default_cc_opt_flags = '-Wno-sign-compare' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 540 | question = ('Please specify optimization flags to use during compilation when' |
| 541 | ' bazel option "--config=opt" is specified [Default is %s]: ' |
| 542 | ) % default_cc_opt_flags |
| 543 | cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS', |
| 544 | question, default_cc_opt_flags) |
| 545 | for opt in cc_opt_flags.split(): |
Michael Case | 0017742 | 2017-11-10 13:14:03 -0800 | [diff] [blame] | 546 | write_to_bazelrc('build:opt --copt=%s' % opt) |
A. Unique TensorFlower | 7e67fdc | 2021-01-12 16:15:11 -0800 | [diff] [blame] | 547 | write_to_bazelrc('build:opt --host_copt=%s' % opt) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 548 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 549 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 550 | def set_tf_cuda_clang(environ_cp): |
| 551 | """set TF_CUDA_CLANG action_env. |
| 552 | |
| 553 | Args: |
| 554 | environ_cp: copy of the os.environ. |
| 555 | """ |
| 556 | question = 'Do you want to use clang as CUDA compiler?' |
| 557 | yes_reply = 'Clang will be used as CUDA compiler.' |
| 558 | no_reply = 'nvcc will be used as CUDA compiler.' |
| 559 | set_action_env_var( |
| 560 | environ_cp, |
| 561 | 'TF_CUDA_CLANG', |
| 562 | None, |
| 563 | False, |
| 564 | question=question, |
| 565 | yes_reply=yes_reply, |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 566 | no_reply=no_reply, |
| 567 | bazel_config_name='cuda_clang') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 568 | |
| 569 | |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 570 | def set_tf_download_clang(environ_cp): |
| 571 | """Set TF_DOWNLOAD_CLANG action_env.""" |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 572 | question = 'Do you wish to download a fresh release of clang? (Experimental)' |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 573 | yes_reply = 'Clang will be downloaded and used to compile tensorflow.' |
| 574 | no_reply = 'Clang will not be downloaded.' |
| 575 | set_action_env_var( |
| 576 | environ_cp, |
| 577 | 'TF_DOWNLOAD_CLANG', |
| 578 | None, |
| 579 | False, |
| 580 | question=question, |
| 581 | yes_reply=yes_reply, |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 582 | no_reply=no_reply, |
| 583 | bazel_config_name='download_clang') |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 584 | |
| 585 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 586 | def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var, |
| 587 | var_default): |
| 588 | """Get var_name either from env, or user or default. |
| 589 | |
| 590 | If var_name has been set as environment variable, use the preset value, else |
| 591 | ask for user input. If no input is provided, the default is used. |
| 592 | |
| 593 | Args: |
| 594 | environ_cp: copy of the os.environ. |
R S Nikhil Krishna | 05e348b | 2019-04-18 15:58:12 +0530 | [diff] [blame] | 595 | var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 596 | ask_for_var: string for how to ask for user input. |
| 597 | var_default: default value string. |
| 598 | |
| 599 | Returns: |
| 600 | string value for var_name |
| 601 | """ |
| 602 | var = environ_cp.get(var_name) |
| 603 | if not var: |
| 604 | var = get_input(ask_for_var) |
Michael Case | d94271a | 2017-08-22 17:26:52 -0700 | [diff] [blame] | 605 | print('\n') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 606 | if not var: |
| 607 | var = var_default |
| 608 | return var |
| 609 | |
| 610 | |
| 611 | def set_clang_cuda_compiler_path(environ_cp): |
| 612 | """Set CLANG_CUDA_COMPILER_PATH.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 613 | default_clang_path = which('clang') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 614 | ask_clang_path = ('Please specify which clang should be used as device and ' |
| 615 | 'host compiler. [Default is %s]: ') % default_clang_path |
| 616 | |
| 617 | while True: |
| 618 | clang_cuda_compiler_path = get_from_env_or_user_or_default( |
| 619 | environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path, |
| 620 | default_clang_path) |
| 621 | if os.path.exists(clang_cuda_compiler_path): |
| 622 | break |
| 623 | |
| 624 | # Reset and retry |
| 625 | print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path) |
| 626 | environ_cp['CLANG_CUDA_COMPILER_PATH'] = '' |
| 627 | |
| 628 | # Set CLANG_CUDA_COMPILER_PATH |
| 629 | environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path |
| 630 | write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH', |
| 631 | clang_cuda_compiler_path) |
| 632 | |
| 633 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 634 | def prompt_loop_or_load_from_env(environ_cp, |
| 635 | var_name, |
| 636 | var_default, |
| 637 | ask_for_var, |
| 638 | check_success, |
| 639 | error_msg, |
| 640 | suppress_default_error=False, |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 641 | resolve_symlinks=False, |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 642 | n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS): |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 643 | """Loop over user prompts for an ENV param until receiving a valid response. |
| 644 | |
| 645 | For the env param var_name, read from the environment or verify user input |
| 646 | until receiving valid input. When done, set var_name in the environ_cp to its |
| 647 | new value. |
| 648 | |
| 649 | Args: |
| 650 | environ_cp: (Dict) copy of the os.environ. |
| 651 | var_name: (String) string for name of environment variable, e.g. "TF_MYVAR". |
| 652 | var_default: (String) default value string. |
| 653 | ask_for_var: (String) string for how to ask for user input. |
| 654 | check_success: (Function) function that takes one argument and returns a |
| 655 | boolean. Should return True if the value provided is considered valid. May |
| 656 | contain a complex error message if error_msg does not provide enough |
| 657 | information. In that case, set suppress_default_error to True. |
| 658 | error_msg: (String) String with one and only one '%s'. Formatted with each |
| 659 | invalid response upon check_success(input) failure. |
| 660 | suppress_default_error: (Bool) Suppress the above error message in favor of |
| 661 | one from the check_success function. |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 662 | resolve_symlinks: (Bool) Translate symbolic links into the real filepath. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 663 | n_ask_attempts: (Integer) Number of times to query for valid input before |
| 664 | raising an error and quitting. |
| 665 | |
| 666 | Returns: |
| 667 | [String] The value of var_name after querying for input. |
| 668 | |
| 669 | Raises: |
| 670 | UserInputError: if a query has been attempted n_ask_attempts times without |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 671 | success, assume that the user has made a scripting error, and will |
| 672 | continue to provide invalid input. Raise the error to avoid infinitely |
| 673 | looping. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 674 | """ |
| 675 | default = environ_cp.get(var_name) or var_default |
| 676 | full_query = '%s [Default is %s]: ' % ( |
| 677 | ask_for_var, |
| 678 | default, |
| 679 | ) |
| 680 | |
| 681 | for _ in range(n_ask_attempts): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 682 | val = get_from_env_or_user_or_default(environ_cp, var_name, full_query, |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 683 | default) |
| 684 | if check_success(val): |
| 685 | break |
| 686 | if not suppress_default_error: |
| 687 | print(error_msg % val) |
| 688 | environ_cp[var_name] = '' |
| 689 | else: |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 690 | raise UserInputError('Invalid %s setting was provided %d times in a row. ' |
| 691 | 'Assuming to be a scripting mistake.' % |
| 692 | (var_name, n_ask_attempts)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 693 | |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 694 | if resolve_symlinks and os.path.islink(val): |
| 695 | val = os.path.realpath(val) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 696 | environ_cp[var_name] = val |
| 697 | return val |
| 698 | |
| 699 | |
| 700 | def create_android_ndk_rule(environ_cp): |
| 701 | """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule.""" |
| 702 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 703 | default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' % |
| 704 | environ_cp['APPDATA']) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 705 | elif is_macos(): |
| 706 | default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 707 | else: |
| 708 | default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 709 | |
| 710 | def valid_ndk_path(path): |
| 711 | return (os.path.exists(path) and |
| 712 | os.path.exists(os.path.join(path, 'source.properties'))) |
| 713 | |
| 714 | android_ndk_home_path = prompt_loop_or_load_from_env( |
| 715 | environ_cp, |
| 716 | var_name='ANDROID_NDK_HOME', |
| 717 | var_default=default_ndk_path, |
| 718 | ask_for_var='Please specify the home path of the Android NDK to use.', |
| 719 | check_success=valid_ndk_path, |
| 720 | error_msg=('The path %s or its child file "source.properties" ' |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 721 | 'does not exist.')) |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 722 | write_action_env_to_bazelrc('ANDROID_NDK_HOME', android_ndk_home_path) |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 723 | write_action_env_to_bazelrc( |
| 724 | 'ANDROID_NDK_API_LEVEL', |
| 725 | get_ndk_api_level(environ_cp, android_ndk_home_path)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 726 | |
| 727 | |
| 728 | def create_android_sdk_rule(environ_cp): |
| 729 | """Set Android variables and write Android SDK WORKSPACE rule.""" |
| 730 | if is_windows() or is_cygwin(): |
| 731 | default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA']) |
| 732 | elif is_macos(): |
Shashi Shekhar | c0ff0cc | 2018-07-17 09:00:24 -0700 | [diff] [blame] | 733 | default_sdk_path = '%s/library/Android/Sdk' % environ_cp['HOME'] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 734 | else: |
| 735 | default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME'] |
| 736 | |
| 737 | def valid_sdk_path(path): |
| 738 | return (os.path.exists(path) and |
| 739 | os.path.exists(os.path.join(path, 'platforms')) and |
| 740 | os.path.exists(os.path.join(path, 'build-tools'))) |
| 741 | |
| 742 | android_sdk_home_path = prompt_loop_or_load_from_env( |
| 743 | environ_cp, |
| 744 | var_name='ANDROID_SDK_HOME', |
| 745 | var_default=default_sdk_path, |
| 746 | ask_for_var='Please specify the home path of the Android SDK to use.', |
| 747 | check_success=valid_sdk_path, |
| 748 | error_msg=('Either %s does not exist, or it does not contain the ' |
| 749 | 'subdirectories "platforms" and "build-tools".')) |
| 750 | |
| 751 | platforms = os.path.join(android_sdk_home_path, 'platforms') |
| 752 | api_levels = sorted(os.listdir(platforms)) |
| 753 | api_levels = [x.replace('android-', '') for x in api_levels] |
| 754 | |
| 755 | def valid_api_level(api_level): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 756 | return os.path.exists( |
| 757 | os.path.join(android_sdk_home_path, 'platforms', |
| 758 | 'android-' + api_level)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 759 | |
| 760 | android_api_level = prompt_loop_or_load_from_env( |
| 761 | environ_cp, |
| 762 | var_name='ANDROID_API_LEVEL', |
| 763 | var_default=api_levels[-1], |
| 764 | ask_for_var=('Please specify the Android SDK API level to use. ' |
| 765 | '[Available levels: %s]') % api_levels, |
| 766 | check_success=valid_api_level, |
| 767 | error_msg='Android-%s is not present in the SDK path.') |
| 768 | |
| 769 | build_tools = os.path.join(android_sdk_home_path, 'build-tools') |
| 770 | versions = sorted(os.listdir(build_tools)) |
| 771 | |
| 772 | def valid_build_tools(version): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 773 | return os.path.exists( |
| 774 | os.path.join(android_sdk_home_path, 'build-tools', version)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 775 | |
| 776 | android_build_tools_version = prompt_loop_or_load_from_env( |
| 777 | environ_cp, |
| 778 | var_name='ANDROID_BUILD_TOOLS_VERSION', |
| 779 | var_default=versions[-1], |
| 780 | ask_for_var=('Please specify an Android build tools version to use. ' |
| 781 | '[Available versions: %s]') % versions, |
| 782 | check_success=valid_build_tools, |
| 783 | error_msg=('The selected SDK does not have build-tools version %s ' |
| 784 | 'available.')) |
| 785 | |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 786 | write_action_env_to_bazelrc('ANDROID_BUILD_TOOLS_VERSION', |
| 787 | android_build_tools_version) |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 788 | write_action_env_to_bazelrc('ANDROID_SDK_API_LEVEL', android_api_level) |
| 789 | write_action_env_to_bazelrc('ANDROID_SDK_HOME', android_sdk_home_path) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 790 | |
| 791 | |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 792 | def get_ndk_api_level(environ_cp, android_ndk_home_path): |
| 793 | """Gets the appropriate NDK API level to use for the provided Android NDK path.""" |
| 794 | |
| 795 | # First check to see if we're using a blessed version of the NDK. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 796 | properties_path = '%s/source.properties' % android_ndk_home_path |
| 797 | if is_windows() or is_cygwin(): |
| 798 | properties_path = cygpath(properties_path) |
| 799 | with open(properties_path, 'r') as f: |
| 800 | filedata = f.read() |
| 801 | |
| 802 | revision = re.search(r'Pkg.Revision = (\d+)', filedata) |
| 803 | if revision: |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 804 | ndk_version = revision.group(1) |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 805 | else: |
| 806 | raise Exception('Unable to parse NDK revision.') |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 807 | if int(ndk_version) not in _SUPPORTED_ANDROID_NDK_VERSIONS: |
| 808 | print('WARNING: The NDK version in %s is %s, which is not ' |
| 809 | 'supported by Bazel (officially supported versions: %s). Please use ' |
| 810 | 'another version. Compiling Android targets may result in confusing ' |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 811 | 'errors.\n' % |
| 812 | (android_ndk_home_path, ndk_version, _SUPPORTED_ANDROID_NDK_VERSIONS)) |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 813 | |
| 814 | # Now grab the NDK API level to use. Note that this is different from the |
| 815 | # SDK API level, as the NDK API level is effectively the *min* target SDK |
| 816 | # version. |
| 817 | platforms = os.path.join(android_ndk_home_path, 'platforms') |
| 818 | api_levels = sorted(os.listdir(platforms)) |
| 819 | api_levels = [ |
| 820 | x.replace('android-', '') for x in api_levels if 'android-' in x |
| 821 | ] |
| 822 | |
| 823 | def valid_api_level(api_level): |
| 824 | return os.path.exists( |
| 825 | os.path.join(android_ndk_home_path, 'platforms', |
| 826 | 'android-' + api_level)) |
| 827 | |
| 828 | android_ndk_api_level = prompt_loop_or_load_from_env( |
| 829 | environ_cp, |
| 830 | var_name='ANDROID_NDK_API_LEVEL', |
Jared Duke | a1309f4 | 2019-11-15 17:09:35 -0800 | [diff] [blame] | 831 | var_default='21', # 21 is required for ARM64 support. |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 832 | ask_for_var=('Please specify the (min) Android NDK API level to use. ' |
| 833 | '[Available levels: %s]') % api_levels, |
| 834 | check_success=valid_api_level, |
| 835 | error_msg='Android-%s is not present in the NDK path.') |
| 836 | |
| 837 | return android_ndk_api_level |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 838 | |
| 839 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 840 | def set_gcc_host_compiler_path(environ_cp): |
| 841 | """Set GCC_HOST_COMPILER_PATH.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 842 | default_gcc_host_compiler_path = which('gcc') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 843 | cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH') |
| 844 | |
| 845 | if os.path.islink(cuda_bin_symlink): |
| 846 | # os.readlink is only available in linux |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 847 | default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 848 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 849 | gcc_host_compiler_path = prompt_loop_or_load_from_env( |
| 850 | environ_cp, |
| 851 | var_name='GCC_HOST_COMPILER_PATH', |
| 852 | var_default=default_gcc_host_compiler_path, |
Ian Hua | 6b51112 | 2021-08-11 21:38:50 +0100 | [diff] [blame] | 853 | ask_for_var='Please specify which gcc should be used by nvcc as the host ' |
| 854 | 'compiler.', |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 855 | check_success=os.path.exists, |
Austin Anderson | 58b236b | 2019-10-22 10:36:10 -0700 | [diff] [blame] | 856 | resolve_symlinks=True, |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 857 | error_msg='Invalid gcc path. %s cannot be found.', |
| 858 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 859 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 860 | write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path) |
| 861 | |
| 862 | |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 863 | def reformat_version_sequence(version_str, sequence_count): |
| 864 | """Reformat the version string to have the given number of sequences. |
| 865 | |
| 866 | For example: |
| 867 | Given (7, 2) -> 7.0 |
| 868 | (7.0.1, 2) -> 7.0 |
| 869 | (5, 1) -> 5 |
| 870 | (5.0.3.2, 1) -> 5 |
| 871 | |
| 872 | Args: |
| 873 | version_str: String, the version string. |
| 874 | sequence_count: int, an integer. |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 875 | |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 876 | Returns: |
| 877 | string, reformatted version string. |
| 878 | """ |
| 879 | v = version_str.split('.') |
| 880 | if len(v) < sequence_count: |
| 881 | v = v + (['0'] * (sequence_count - len(v))) |
| 882 | |
| 883 | return '.'.join(v[:sequence_count]) |
| 884 | |
| 885 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 886 | def set_tf_cuda_paths(environ_cp): |
| 887 | """Set TF_CUDA_PATHS.""" |
| 888 | ask_cuda_paths = ( |
| 889 | 'Please specify the comma-separated list of base paths to look for CUDA ' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 890 | 'libraries and headers. [Leave empty to use the default]: ') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 891 | tf_cuda_paths = get_from_env_or_user_or_default(environ_cp, 'TF_CUDA_PATHS', |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 892 | ask_cuda_paths, '') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 893 | if tf_cuda_paths: |
| 894 | environ_cp['TF_CUDA_PATHS'] = tf_cuda_paths |
| 895 | |
| 896 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 897 | def set_tf_cuda_version(environ_cp): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 898 | """Set TF_CUDA_VERSION.""" |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 899 | ask_cuda_version = ( |
| 900 | 'Please specify the CUDA SDK version you want to use. ' |
| 901 | '[Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 902 | tf_cuda_version = get_from_env_or_user_or_default(environ_cp, |
| 903 | 'TF_CUDA_VERSION', |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 904 | ask_cuda_version, |
| 905 | _DEFAULT_CUDA_VERSION) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 906 | environ_cp['TF_CUDA_VERSION'] = tf_cuda_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 907 | |
| 908 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 909 | def set_tf_cudnn_version(environ_cp): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 910 | """Set TF_CUDNN_VERSION.""" |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 911 | ask_cudnn_version = ( |
| 912 | 'Please specify the cuDNN version you want to use. ' |
| 913 | '[Leave empty to default to cuDNN %s]: ') % _DEFAULT_CUDNN_VERSION |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 914 | tf_cudnn_version = get_from_env_or_user_or_default(environ_cp, |
| 915 | 'TF_CUDNN_VERSION', |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 916 | ask_cudnn_version, |
| 917 | _DEFAULT_CUDNN_VERSION) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 918 | environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 919 | |
| 920 | |
Mingxing Tan | 1e7b0e4 | 2018-06-28 19:13:20 -0700 | [diff] [blame] | 921 | def is_cuda_compatible(lib, cuda_ver, cudnn_ver): |
| 922 | """Check compatibility between given library and cudnn/cudart libraries.""" |
| 923 | ldd_bin = which('ldd') or '/usr/bin/ldd' |
| 924 | ldd_out = run_shell([ldd_bin, lib], True) |
| 925 | ldd_out = ldd_out.split(os.linesep) |
| 926 | cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$') |
| 927 | cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$') |
| 928 | cudnn = None |
| 929 | cudart = None |
| 930 | cudnn_ok = True # assume no cudnn dependency by default |
| 931 | cuda_ok = True # assume no cuda dependency by default |
| 932 | for line in ldd_out: |
| 933 | if 'libcudnn.so' in line: |
| 934 | cudnn = cudnn_pattern.search(line) |
| 935 | cudnn_ok = False |
| 936 | elif 'libcudart.so' in line: |
| 937 | cudart = cuda_pattern.search(line) |
| 938 | cuda_ok = False |
| 939 | if cudnn and len(cudnn.group(1)): |
| 940 | cudnn = convert_version_to_int(cudnn.group(1)) |
| 941 | if cudart and len(cudart.group(1)): |
| 942 | cudart = convert_version_to_int(cudart.group(1)) |
| 943 | if cudnn is not None: |
| 944 | cudnn_ok = (cudnn == cudnn_ver) |
| 945 | if cudart is not None: |
| 946 | cuda_ok = (cudart == cuda_ver) |
| 947 | return cudnn_ok and cuda_ok |
| 948 | |
| 949 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 950 | def set_tf_tensorrt_version(environ_cp): |
| 951 | """Set TF_TENSORRT_VERSION.""" |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 952 | if not is_linux(): |
| 953 | raise ValueError('Currently TensorRT is only supported on Linux platform.') |
| 954 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 955 | if not int(environ_cp.get('TF_NEED_TENSORRT', False)): |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 956 | return |
| 957 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 958 | ask_tensorrt_version = ( |
| 959 | 'Please specify the TensorRT version you want to use. ' |
Guangda Lai | faa93ac | 2019-10-23 15:07:12 -0700 | [diff] [blame] | 960 | '[Leave empty to default to TensorRT %s]: ') % _DEFAULT_TENSORRT_VERSION |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 961 | tf_tensorrt_version = get_from_env_or_user_or_default( |
| 962 | environ_cp, 'TF_TENSORRT_VERSION', ask_tensorrt_version, |
| 963 | _DEFAULT_TENSORRT_VERSION) |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 964 | environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 965 | |
| 966 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 967 | def set_tf_nccl_version(environ_cp): |
| 968 | """Set TF_NCCL_VERSION.""" |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 969 | if not is_linux(): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 970 | raise ValueError('Currently NCCL is only supported on Linux platform.') |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 971 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 972 | if 'TF_NCCL_VERSION' in environ_cp: |
| 973 | return |
| 974 | |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 975 | ask_nccl_version = ( |
A. Unique TensorFlower | 53faa31 | 2018-10-05 08:46:54 -0700 | [diff] [blame] | 976 | 'Please specify the locally installed NCCL version you want to use. ' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 977 | '[Leave empty to use http://github.com/nvidia/nccl]: ') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 978 | tf_nccl_version = get_from_env_or_user_or_default(environ_cp, |
| 979 | 'TF_NCCL_VERSION', |
| 980 | ask_nccl_version, '') |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 981 | environ_cp['TF_NCCL_VERSION'] = tf_nccl_version |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 982 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 983 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 984 | def get_native_cuda_compute_capabilities(environ_cp): |
| 985 | """Get native cuda compute capabilities. |
| 986 | |
| 987 | Args: |
| 988 | environ_cp: copy of the os.environ. |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 989 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 990 | Returns: |
| 991 | string of native cuda compute capabilities, separated by comma. |
| 992 | """ |
| 993 | device_query_bin = os.path.join( |
| 994 | environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery') |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 995 | if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK): |
| 996 | try: |
| 997 | output = run_shell(device_query_bin).split('\n') |
| 998 | pattern = re.compile('[0-9]*\\.[0-9]*') |
| 999 | output = [pattern.search(x) for x in output if 'Capability' in x] |
| 1000 | output = ','.join(x.group() for x in output if x is not None) |
| 1001 | except subprocess.CalledProcessError: |
| 1002 | output = '' |
| 1003 | else: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1004 | output = '' |
| 1005 | return output |
| 1006 | |
| 1007 | |
| 1008 | def set_tf_cuda_compute_capabilities(environ_cp): |
| 1009 | """Set TF_CUDA_COMPUTE_CAPABILITIES.""" |
| 1010 | while True: |
| 1011 | native_cuda_compute_capabilities = get_native_cuda_compute_capabilities( |
| 1012 | environ_cp) |
| 1013 | if not native_cuda_compute_capabilities: |
| 1014 | default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES |
| 1015 | else: |
| 1016 | default_cuda_compute_capabilities = native_cuda_compute_capabilities |
| 1017 | |
| 1018 | ask_cuda_compute_capabilities = ( |
Amit Patankar | a5393e9 | 2020-05-28 14:29:56 -0700 | [diff] [blame] | 1019 | 'Please specify a list of comma-separated CUDA compute capabilities ' |
| 1020 | 'you want to build with.\nYou can find the compute capability of your ' |
| 1021 | 'device at: https://developer.nvidia.com/cuda-gpus. Each capability ' |
| 1022 | 'can be specified as "x.y" or "compute_xy" to include both virtual and' |
| 1023 | ' binary GPU code, or as "sm_xy" to only include the binary ' |
| 1024 | 'code.\nPlease note that each additional compute capability ' |
| 1025 | 'significantly increases your build time and binary size, and that ' |
| 1026 | 'TensorFlow only supports compute capabilities >= 3.5 [Default is: ' |
| 1027 | '%s]: ' % default_cuda_compute_capabilities) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1028 | tf_cuda_compute_capabilities = get_from_env_or_user_or_default( |
| 1029 | environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES', |
| 1030 | ask_cuda_compute_capabilities, default_cuda_compute_capabilities) |
| 1031 | # Check whether all capabilities from the input is valid |
| 1032 | all_valid = True |
Maciej | d0f5bc1 | 2018-04-30 22:30:58 -0500 | [diff] [blame] | 1033 | # Remove all whitespace characters before splitting the string |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 1034 | # that users may insert by accident, as this will result in error |
Maciej | d0f5bc1 | 2018-04-30 22:30:58 -0500 | [diff] [blame] | 1035 | tf_cuda_compute_capabilities = ''.join(tf_cuda_compute_capabilities.split()) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1036 | for compute_capability in tf_cuda_compute_capabilities.split(','): |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1037 | m = re.match('[0-9]+.[0-9]+', compute_capability) |
| 1038 | if not m: |
Amit Patankar | a5393e9 | 2020-05-28 14:29:56 -0700 | [diff] [blame] | 1039 | # We now support sm_35,sm_50,sm_60,compute_70. |
| 1040 | sm_compute_match = re.match('(sm|compute)_?([0-9]+[0-9]+)', |
| 1041 | compute_capability) |
| 1042 | if not sm_compute_match: |
| 1043 | print('Invalid compute capability: %s' % compute_capability) |
| 1044 | all_valid = False |
| 1045 | else: |
Amit Patankar | 639d937 | 2020-05-30 01:09:36 -0700 | [diff] [blame] | 1046 | ver = int(sm_compute_match.group(2)) |
Amit Patankar | a5393e9 | 2020-05-28 14:29:56 -0700 | [diff] [blame] | 1047 | if ver < 30: |
| 1048 | print( |
| 1049 | 'ERROR: TensorFlow only supports small CUDA compute' |
| 1050 | ' capabilities of sm_30 and higher. Please re-specify the list' |
| 1051 | ' of compute capabilities excluding version %s.' % ver) |
| 1052 | all_valid = False |
| 1053 | if ver < 35: |
| 1054 | print('WARNING: XLA does not support CUDA compute capabilities ' |
| 1055 | 'lower than sm_35. Disable XLA when running on older GPUs.') |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1056 | else: |
P Sudeepam | 5209356 | 2019-02-17 17:34:01 +0530 | [diff] [blame] | 1057 | ver = float(m.group(0)) |
A. Unique TensorFlower | 8dc2d0e | 2019-03-12 01:41:05 -0700 | [diff] [blame] | 1058 | if ver < 3.0: |
| 1059 | print('ERROR: TensorFlow only supports CUDA compute capabilities 3.0 ' |
Austin Anderson | 32202dc | 2019-02-19 10:46:27 -0800 | [diff] [blame] | 1060 | 'and higher. Please re-specify the list of compute ' |
| 1061 | 'capabilities excluding version %s.' % ver) |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1062 | all_valid = False |
A. Unique TensorFlower | 8dc2d0e | 2019-03-12 01:41:05 -0700 | [diff] [blame] | 1063 | if ver < 3.5: |
| 1064 | print('WARNING: XLA does not support CUDA compute capabilities ' |
| 1065 | 'lower than 3.5. Disable XLA when running on older GPUs.') |
P Sudeepam | 765ceda | 2019-02-17 17:39:08 +0530 | [diff] [blame] | 1066 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1067 | if all_valid: |
| 1068 | break |
| 1069 | |
| 1070 | # Reset and Retry |
| 1071 | environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = '' |
| 1072 | |
| 1073 | # Set TF_CUDA_COMPUTE_CAPABILITIES |
| 1074 | environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities |
| 1075 | write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES', |
| 1076 | tf_cuda_compute_capabilities) |
| 1077 | |
| 1078 | |
| 1079 | def set_other_cuda_vars(environ_cp): |
| 1080 | """Set other CUDA related variables.""" |
A. Unique TensorFlower | ab39198 | 2018-07-11 04:52:49 -0700 | [diff] [blame] | 1081 | # If CUDA is enabled, always use GPU during build and test. |
| 1082 | if environ_cp.get('TF_CUDA_CLANG') == '1': |
| 1083 | write_to_bazelrc('build --config=cuda_clang') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1084 | else: |
A. Unique TensorFlower | ab39198 | 2018-07-11 04:52:49 -0700 | [diff] [blame] | 1085 | write_to_bazelrc('build --config=cuda') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1086 | |
| 1087 | |
| 1088 | def set_host_cxx_compiler(environ_cp): |
| 1089 | """Set HOST_CXX_COMPILER.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1090 | default_cxx_host_compiler = which('g++') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1091 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1092 | host_cxx_compiler = prompt_loop_or_load_from_env( |
| 1093 | environ_cp, |
| 1094 | var_name='HOST_CXX_COMPILER', |
| 1095 | var_default=default_cxx_host_compiler, |
| 1096 | ask_for_var=('Please specify which C++ compiler should be used as the ' |
| 1097 | 'host C++ compiler.'), |
| 1098 | check_success=os.path.exists, |
| 1099 | error_msg='Invalid C++ compiler path. %s cannot be found.', |
| 1100 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1101 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1102 | write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler) |
| 1103 | |
| 1104 | |
| 1105 | def set_host_c_compiler(environ_cp): |
| 1106 | """Set HOST_C_COMPILER.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1107 | default_c_host_compiler = which('gcc') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1108 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1109 | host_c_compiler = prompt_loop_or_load_from_env( |
| 1110 | environ_cp, |
| 1111 | var_name='HOST_C_COMPILER', |
| 1112 | var_default=default_c_host_compiler, |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1113 | ask_for_var=('Please specify which C compiler should be used as the host ' |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1114 | 'C compiler.'), |
| 1115 | check_success=os.path.exists, |
| 1116 | error_msg='Invalid C compiler path. %s cannot be found.', |
| 1117 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1118 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1119 | write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler) |
| 1120 | |
| 1121 | |
William D. Irons | 391b21e | 2020-02-11 22:29:20 +0000 | [diff] [blame] | 1122 | def system_specific_test_config(environ_cp): |
A. Unique TensorFlower | 7bd8637 | 2019-03-21 15:19:30 -0700 | [diff] [blame] | 1123 | """Add default build and test flags required for TF tests to bazelrc.""" |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1124 | write_to_bazelrc('test --flaky_test_attempts=3') |
| 1125 | write_to_bazelrc('test --test_size_filters=small,medium') |
frreiss | 8b013a7 | 2019-11-20 11:36:27 -0800 | [diff] [blame] | 1126 | |
| 1127 | # Each instance of --test_tag_filters or --build_tag_filters overrides all |
| 1128 | # previous instances, so we need to build up a complete list and write a |
| 1129 | # single list of filters for the .bazelrc file. |
| 1130 | |
| 1131 | # Filters to use with both --test_tag_filters and --build_tag_filters |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1132 | test_and_build_filters = ['-benchmark-test', '-no_oss'] |
| 1133 | # Additional filters for --test_tag_filters beyond those in |
| 1134 | # test_and_build_filters |
| 1135 | test_only_filters = ['-oss_serial'] |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1136 | if is_windows(): |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1137 | test_and_build_filters.append('-no_windows') |
Deven Desai | f0bf514 | 2020-04-17 15:00:56 +0000 | [diff] [blame] | 1138 | if ((environ_cp.get('TF_NEED_CUDA', None) == '1') or |
| 1139 | (environ_cp.get('TF_NEED_ROCM', None) == '1')): |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1140 | test_and_build_filters += ['-no_windows_gpu', '-no_gpu'] |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1141 | else: |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1142 | test_and_build_filters.append('-gpu') |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1143 | elif is_macos(): |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1144 | test_and_build_filters += ['-gpu', '-nomac', '-no_mac'] |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1145 | elif is_linux(): |
Deven Desai | f0bf514 | 2020-04-17 15:00:56 +0000 | [diff] [blame] | 1146 | if ((environ_cp.get('TF_NEED_CUDA', None) == '1') or |
| 1147 | (environ_cp.get('TF_NEED_ROCM', None) == '1')): |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1148 | test_and_build_filters.append('-no_gpu') |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1149 | write_to_bazelrc('test --test_env=LD_LIBRARY_PATH') |
| 1150 | else: |
frreiss | bab7d7d | 2019-11-21 10:52:55 -0800 | [diff] [blame] | 1151 | test_and_build_filters.append('-gpu') |
frreiss | 3adf417 | 2019-12-06 14:16:49 -0800 | [diff] [blame] | 1152 | |
| 1153 | # Disable tests with "v1only" tag in "v2" Bazel config, but not in "v1" config |
| 1154 | write_to_bazelrc('test:v1 --test_tag_filters=%s' % |
TensorFlower Gardener | e515908 | 2019-11-22 11:47:38 -0800 | [diff] [blame] | 1155 | ','.join(test_and_build_filters + test_only_filters)) |
frreiss | 3adf417 | 2019-12-06 14:16:49 -0800 | [diff] [blame] | 1156 | write_to_bazelrc('test:v1 --build_tag_filters=%s' % |
TensorFlower Gardener | e515908 | 2019-11-22 11:47:38 -0800 | [diff] [blame] | 1157 | ','.join(test_and_build_filters)) |
TensorFlower Gardener | 26a1de3 | 2019-12-17 07:11:48 -0800 | [diff] [blame] | 1158 | write_to_bazelrc( |
| 1159 | 'test:v2 --test_tag_filters=%s' % |
| 1160 | ','.join(test_and_build_filters + test_only_filters + ['-v1only'])) |
frreiss | 3adf417 | 2019-12-06 14:16:49 -0800 | [diff] [blame] | 1161 | write_to_bazelrc('test:v2 --build_tag_filters=%s' % |
TensorFlower Gardener | 26a1de3 | 2019-12-17 07:11:48 -0800 | [diff] [blame] | 1162 | ','.join(test_and_build_filters + ['-v1only'])) |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1163 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1164 | |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1165 | def set_system_libs_flag(environ_cp): |
| 1166 | syslibs = environ_cp.get('TF_SYSTEM_LIBS', '') |
TensorFlower Gardener | 61a8720 | 2018-10-01 12:25:39 -0700 | [diff] [blame] | 1167 | if syslibs: |
Jason Zaman | 5fc39bd | 2018-09-16 01:38:55 +0800 | [diff] [blame] | 1168 | if ',' in syslibs: |
| 1169 | syslibs = ','.join(sorted(syslibs.split(','))) |
| 1170 | else: |
| 1171 | syslibs = ','.join(sorted(syslibs.split())) |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1172 | write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs) |
| 1173 | |
Alexander Grund | 92214e9 | 2020-10-19 15:35:43 +0200 | [diff] [blame] | 1174 | for varname in ('PREFIX', 'LIBDIR', 'INCLUDEDIR', 'PROTOBUF_INCLUDE_PATH'): |
| 1175 | if varname in environ_cp: |
| 1176 | write_to_bazelrc('build --define=%s=%s' % (varname, environ_cp[varname])) |
Jason Zaman | 5fc39bd | 2018-09-16 01:38:55 +0800 | [diff] [blame] | 1177 | |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1178 | |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1179 | def set_windows_build_flags(environ_cp): |
| 1180 | """Set Windows specific build options.""" |
Austin Anderson | 9983bb3 | 2020-11-12 16:39:52 -0800 | [diff] [blame] | 1181 | |
| 1182 | # First available in VS 16.4. Speeds up Windows compile times by a lot. See |
| 1183 | # https://groups.google.com/a/tensorflow.org/d/topic/build/SsW98Eo7l3o/discussion |
| 1184 | # pylint: disable=line-too-long |
TensorFlower Gardener | 5b77c61 | 2020-12-22 06:19:25 -0800 | [diff] [blame] | 1185 | write_to_bazelrc( |
| 1186 | 'build --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions' |
| 1187 | ) |
Austin Anderson | 6d7c376 | 2019-11-19 17:04:19 -0800 | [diff] [blame] | 1188 | |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1189 | if get_var( |
| 1190 | environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline', |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1191 | True, ('Would you like to override eigen strong inline for some C++ ' |
| 1192 | 'compilation to reduce the compilation time?'), |
| 1193 | 'Eigen strong inline overridden.', 'Not overriding eigen strong inline, ' |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1194 | 'some compilations could take more than 20 mins.'): |
| 1195 | # Due to a known MSVC compiler issue |
| 1196 | # https://github.com/tensorflow/tensorflow/issues/10521 |
| 1197 | # Overriding eigen strong inline speeds up the compiling of |
| 1198 | # conv_grad_ops_3d.cc and conv_ops_3d.cc by 20 minutes, |
| 1199 | # but this also hurts the performance. Let users decide what they want. |
| 1200 | write_to_bazelrc('build --define=override_eigen_strong_inline=true') |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1201 | |
A. Unique TensorFlower | 061c359 | 2017-11-13 14:21:04 -0800 | [diff] [blame] | 1202 | |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1203 | def config_info_line(name, help_text): |
| 1204 | """Helper function to print formatted help text for Bazel config options.""" |
| 1205 | print('\t--config=%-12s\t# %s' % (name, help_text)) |
| 1206 | |
| 1207 | |
Ian Hua | 6b51112 | 2021-08-11 21:38:50 +0100 | [diff] [blame] | 1208 | def configure_ios(environ_cp): |
| 1209 | """Configures TensorFlow for iOS builds.""" |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1210 | if not is_macos(): |
| 1211 | return |
Ian Hua | 6b51112 | 2021-08-11 21:38:50 +0100 | [diff] [blame] | 1212 | if not get_var(environ_cp, 'TF_CONFIGURE_IOS', 'iOS', False): |
| 1213 | return |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1214 | for filepath in APPLE_BAZEL_FILES: |
| 1215 | existing_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath + '.apple') |
| 1216 | renamed_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath) |
| 1217 | symlink_force(existing_filepath, renamed_filepath) |
| 1218 | for filepath in IOS_FILES: |
| 1219 | filename = os.path.basename(filepath) |
| 1220 | new_filepath = os.path.join(_TF_WORKSPACE_ROOT, filename) |
| 1221 | symlink_force(filepath, new_filepath) |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1222 | |
| 1223 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1224 | def validate_cuda_config(environ_cp): |
| 1225 | """Run find_cuda_config.py and return cuda_toolkit_path, or None.""" |
| 1226 | |
| 1227 | def maybe_encode_env(env): |
| 1228 | """Encodes unicode in env to str on Windows python 2.x.""" |
| 1229 | if not is_windows() or sys.version_info[0] != 2: |
| 1230 | return env |
| 1231 | for k, v in env.items(): |
| 1232 | if isinstance(k, unicode): |
| 1233 | k = k.encode('ascii') |
| 1234 | if isinstance(v, unicode): |
| 1235 | v = v.encode('ascii') |
| 1236 | env[k] = v |
| 1237 | return env |
| 1238 | |
| 1239 | cuda_libraries = ['cuda', 'cudnn'] |
| 1240 | if is_linux(): |
A. Unique TensorFlower | 28fc9cc | 2019-05-01 14:17:54 -0700 | [diff] [blame] | 1241 | if int(environ_cp.get('TF_NEED_TENSORRT', False)): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1242 | cuda_libraries.append('tensorrt') |
A. Unique TensorFlower | b2e7f67 | 2019-04-30 09:20:36 -0700 | [diff] [blame] | 1243 | if environ_cp.get('TF_NCCL_VERSION', None): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1244 | cuda_libraries.append('nccl') |
| 1245 | |
Austin Anderson | b1f946a | 2021-01-26 11:07:14 -0800 | [diff] [blame] | 1246 | paths = glob.glob('**/third_party/gpus/find_cuda_config.py', recursive=True) |
| 1247 | if not paths: |
| 1248 | raise FileNotFoundError( |
| 1249 | "Can't find 'find_cuda_config.py' script inside working directory") |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1250 | proc = subprocess.Popen( |
Austin Anderson | b1f946a | 2021-01-26 11:07:14 -0800 | [diff] [blame] | 1251 | [environ_cp['PYTHON_BIN_PATH'], paths[0]] + cuda_libraries, |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1252 | stdout=subprocess.PIPE, |
| 1253 | env=maybe_encode_env(environ_cp)) |
| 1254 | |
| 1255 | if proc.wait(): |
| 1256 | # Errors from find_cuda_config.py were sent to stderr. |
A. Unique TensorFlower | 0cb8a07 | 2019-04-16 08:16:39 -0700 | [diff] [blame] | 1257 | print('Asking for detailed CUDA configuration...\n') |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1258 | return False |
| 1259 | |
| 1260 | config = dict( |
| 1261 | tuple(line.decode('ascii').rstrip().split(': ')) for line in proc.stdout) |
| 1262 | |
| 1263 | print('Found CUDA %s in:' % config['cuda_version']) |
| 1264 | print(' %s' % config['cuda_library_dir']) |
| 1265 | print(' %s' % config['cuda_include_dir']) |
| 1266 | |
| 1267 | print('Found cuDNN %s in:' % config['cudnn_version']) |
| 1268 | print(' %s' % config['cudnn_library_dir']) |
| 1269 | print(' %s' % config['cudnn_include_dir']) |
| 1270 | |
| 1271 | if 'tensorrt_version' in config: |
| 1272 | print('Found TensorRT %s in:' % config['tensorrt_version']) |
| 1273 | print(' %s' % config['tensorrt_library_dir']) |
| 1274 | print(' %s' % config['tensorrt_include_dir']) |
| 1275 | |
| 1276 | if config.get('nccl_version', None): |
| 1277 | print('Found NCCL %s in:' % config['nccl_version']) |
| 1278 | print(' %s' % config['nccl_library_dir']) |
| 1279 | print(' %s' % config['nccl_include_dir']) |
| 1280 | |
| 1281 | print('\n') |
| 1282 | |
| 1283 | environ_cp['CUDA_TOOLKIT_PATH'] = config['cuda_toolkit_path'] |
| 1284 | return True |
| 1285 | |
| 1286 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1287 | def main(): |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1288 | global _TF_WORKSPACE_ROOT |
| 1289 | global _TF_BAZELRC |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 1290 | global _TF_CURRENT_BAZEL_VERSION |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1291 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1292 | parser = argparse.ArgumentParser() |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1293 | parser.add_argument( |
| 1294 | '--workspace', |
| 1295 | type=str, |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1296 | default=os.path.abspath(os.path.dirname(__file__)), |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1297 | help='The absolute path to your active Bazel workspace.') |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1298 | args = parser.parse_args() |
| 1299 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1300 | _TF_WORKSPACE_ROOT = args.workspace |
| 1301 | _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME) |
| 1302 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1303 | # Make a copy of os.environ to be clear when functions and getting and setting |
| 1304 | # environment variables. |
| 1305 | environ_cp = dict(os.environ) |
| 1306 | |
George Karpenkov | 07898e7 | 2020-05-20 11:28:10 -0700 | [diff] [blame] | 1307 | try: |
| 1308 | current_bazel_version = check_bazel_version(_TF_MIN_BAZEL_VERSION, |
| 1309 | _TF_MAX_BAZEL_VERSION) |
| 1310 | except subprocess.CalledProcessError as e: |
Amit Patankar | a5393e9 | 2020-05-28 14:29:56 -0700 | [diff] [blame] | 1311 | print('Error checking bazel version: ', e.output.decode('UTF-8').strip()) |
George Karpenkov | 07898e7 | 2020-05-20 11:28:10 -0700 | [diff] [blame] | 1312 | raise e |
| 1313 | |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 1314 | _TF_CURRENT_BAZEL_VERSION = convert_version_to_int(current_bazel_version) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1315 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1316 | reset_tf_configure_bazelrc() |
Yun Peng | 03e63a2 | 2018-11-07 11:18:53 +0100 | [diff] [blame] | 1317 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1318 | cleanup_makefile() |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 1319 | setup_python(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1320 | |
| 1321 | if is_windows(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1322 | environ_cp['TF_NEED_OPENCL'] = '0' |
| 1323 | environ_cp['TF_CUDA_CLANG'] = '0' |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 1324 | environ_cp['TF_NEED_TENSORRT'] = '0' |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1325 | # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on |
| 1326 | # Windows. |
| 1327 | environ_cp['TF_DOWNLOAD_CLANG'] = '0' |
A. Unique TensorFlower | 6e97fb3 | 2018-07-16 14:07:29 -0700 | [diff] [blame] | 1328 | environ_cp['TF_NEED_MPI'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1329 | |
| 1330 | if is_macos(): |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 1331 | environ_cp['TF_NEED_TENSORRT'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1332 | |
Ian Hua | 6b51112 | 2021-08-11 21:38:50 +0100 | [diff] [blame] | 1333 | with_xla_support = environ_cp.get('TF_ENABLE_XLA', None) |
| 1334 | if with_xla_support is not None: |
| 1335 | write_to_bazelrc('build --define=with_xla_support=%s' % |
| 1336 | ('true' if int(with_xla_support) else 'false')) |
Gunhan Gulsoy | f4edbd9 | 2020-02-27 16:07:32 -0800 | [diff] [blame] | 1337 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1338 | set_action_env_var( |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1339 | environ_cp, 'TF_NEED_ROCM', 'ROCm', False, bazel_config_name='rocm') |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1340 | if (environ_cp.get('TF_NEED_ROCM') == '1' and |
TensorFlower Gardener | 62e6016 | 2018-09-27 10:22:55 -0700 | [diff] [blame] | 1341 | 'LD_LIBRARY_PATH' in environ_cp and |
| 1342 | environ_cp.get('LD_LIBRARY_PATH') != '1'): |
| 1343 | write_action_env_to_bazelrc('LD_LIBRARY_PATH', |
| 1344 | environ_cp.get('LD_LIBRARY_PATH')) |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1345 | |
Deven Desai | f0bf514 | 2020-04-17 15:00:56 +0000 | [diff] [blame] | 1346 | if (environ_cp.get('TF_NEED_ROCM') == '1' and environ_cp.get('ROCM_PATH')): |
TensorFlower Gardener | 05fdea6 | 2020-04-23 17:12:38 -0700 | [diff] [blame] | 1347 | write_action_env_to_bazelrc('ROCM_PATH', environ_cp.get('ROCM_PATH')) |
Ian Hua | 6b51112 | 2021-08-11 21:38:50 +0100 | [diff] [blame] | 1348 | write_action_env_to_bazelrc('ROCBLAS_TENSILE_LIBPATH', |
| 1349 | environ_cp.get('ROCM_PATH') + '/lib/library') |
Deven Desai | 4c0b4a8 | 2020-06-30 00:55:05 +0000 | [diff] [blame] | 1350 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1351 | environ_cp['TF_NEED_CUDA'] = str( |
| 1352 | int(get_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False))) |
A. Unique TensorFlower | 24cbb2a | 2017-09-08 07:45:44 -0700 | [diff] [blame] | 1353 | if (environ_cp.get('TF_NEED_CUDA') == '1' and |
| 1354 | 'TF_CUDA_CONFIG_REPO' not in environ_cp): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1355 | |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1356 | set_action_env_var( |
| 1357 | environ_cp, |
| 1358 | 'TF_NEED_TENSORRT', |
| 1359 | 'TensorRT', |
| 1360 | False, |
| 1361 | bazel_config_name='tensorrt') |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1362 | |
| 1363 | environ_save = dict(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1364 | for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1365 | |
| 1366 | if validate_cuda_config(environ_cp): |
| 1367 | cuda_env_names = [ |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1368 | 'TF_CUDA_VERSION', |
| 1369 | 'TF_CUBLAS_VERSION', |
| 1370 | 'TF_CUDNN_VERSION', |
| 1371 | 'TF_TENSORRT_VERSION', |
| 1372 | 'TF_NCCL_VERSION', |
| 1373 | 'TF_CUDA_PATHS', |
A. Unique TensorFlower | b2e7f67 | 2019-04-30 09:20:36 -0700 | [diff] [blame] | 1374 | # Items below are for backwards compatibility when not using |
| 1375 | # TF_CUDA_PATHS. |
A. Unique TensorFlower | fef5167 | 2019-05-23 04:19:20 -0700 | [diff] [blame] | 1376 | 'CUDA_TOOLKIT_PATH', |
| 1377 | 'CUDNN_INSTALL_PATH', |
| 1378 | 'NCCL_INSTALL_PATH', |
| 1379 | 'NCCL_HDR_PATH', |
| 1380 | 'TENSORRT_INSTALL_PATH' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1381 | ] |
A. Unique TensorFlower | b2e7f67 | 2019-04-30 09:20:36 -0700 | [diff] [blame] | 1382 | # Note: set_action_env_var above already writes to bazelrc. |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1383 | for name in cuda_env_names: |
| 1384 | if name in environ_cp: |
| 1385 | write_action_env_to_bazelrc(name, environ_cp[name]) |
| 1386 | break |
| 1387 | |
| 1388 | # Restore settings changed below if CUDA config could not be validated. |
| 1389 | environ_cp = dict(environ_save) |
| 1390 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1391 | set_tf_cuda_version(environ_cp) |
| 1392 | set_tf_cudnn_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1393 | if is_linux(): |
| 1394 | set_tf_tensorrt_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1395 | set_tf_nccl_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1396 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame] | 1397 | set_tf_cuda_paths(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1398 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1399 | else: |
| 1400 | raise UserInputError( |
| 1401 | 'Invalid CUDA setting were provided %d ' |
| 1402 | 'times in a row. Assuming to be a scripting mistake.' % |
| 1403 | _DEFAULT_PROMPT_ASK_ATTEMPTS) |
| 1404 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1405 | set_tf_cuda_compute_capabilities(environ_cp) |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 1406 | if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get( |
| 1407 | 'LD_LIBRARY_PATH') != '1': |
| 1408 | write_action_env_to_bazelrc('LD_LIBRARY_PATH', |
| 1409 | environ_cp.get('LD_LIBRARY_PATH')) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1410 | |
| 1411 | set_tf_cuda_clang(environ_cp) |
| 1412 | if environ_cp.get('TF_CUDA_CLANG') == '1': |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1413 | # Ask whether we should download the clang toolchain. |
| 1414 | set_tf_download_clang(environ_cp) |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 1415 | if environ_cp.get('TF_DOWNLOAD_CLANG') != '1': |
| 1416 | # Set up which clang we should use as the cuda / host compiler. |
| 1417 | set_clang_cuda_compiler_path(environ_cp) |
Ilya Biryukov | 1c3d02e | 2018-09-04 03:09:52 -0700 | [diff] [blame] | 1418 | else: |
| 1419 | # Use downloaded LLD for linking. |
| 1420 | write_to_bazelrc('build:cuda_clang --config=download_clang_use_lld') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1421 | else: |
| 1422 | # Set up which gcc nvcc should use as the host compiler |
| 1423 | # No need to set this on Windows |
| 1424 | if not is_windows(): |
| 1425 | set_gcc_host_compiler_path(environ_cp) |
| 1426 | set_other_cuda_vars(environ_cp) |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1427 | else: |
| 1428 | # CUDA not required. Ask whether we should download the clang toolchain and |
| 1429 | # use it for the CPU build. |
| 1430 | set_tf_download_clang(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1431 | |
Sanjoy Das | 3cbb507 | 2020-09-15 11:02:26 -0700 | [diff] [blame] | 1432 | # ROCm / CUDA are mutually exclusive. |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1433 | # At most 1 GPU platform can be configured. |
| 1434 | gpu_platform_count = 0 |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1435 | if environ_cp.get('TF_NEED_ROCM') == '1': |
| 1436 | gpu_platform_count += 1 |
| 1437 | if environ_cp.get('TF_NEED_CUDA') == '1': |
| 1438 | gpu_platform_count += 1 |
| 1439 | if gpu_platform_count >= 2: |
Sanjoy Das | 3cbb507 | 2020-09-15 11:02:26 -0700 | [diff] [blame] | 1440 | raise UserInputError('CUDA / ROCm are mututally exclusive. ' |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1441 | 'At most 1 GPU platform can be configured.') |
| 1442 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1443 | set_cc_opt_flags(environ_cp) |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1444 | set_system_libs_flag(environ_cp) |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1445 | if is_windows(): |
| 1446 | set_windows_build_flags(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1447 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1448 | if get_var(environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', False, |
| 1449 | ('Would you like to interactively configure ./WORKSPACE for ' |
| 1450 | 'Android builds?'), 'Searching for NDK and SDK installations.', |
| 1451 | 'Not configuring the WORKSPACE for Android builds.'): |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 1452 | create_android_ndk_rule(environ_cp) |
| 1453 | create_android_sdk_rule(environ_cp) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1454 | |
William D. Irons | 391b21e | 2020-02-11 22:29:20 +0000 | [diff] [blame] | 1455 | system_specific_test_config(environ_cp) |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1456 | |
Ian Hua | 6b51112 | 2021-08-11 21:38:50 +0100 | [diff] [blame] | 1457 | configure_ios(environ_cp) |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1458 | |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1459 | print('Preconfigured Bazel build configs. You can use any of the below by ' |
| 1460 | 'adding "--config=<>" to your build command. See .bazelrc for more ' |
| 1461 | 'details.') |
| 1462 | config_info_line('mkl', 'Build with MKL support.') |
Ian Hua | 6b51112 | 2021-08-11 21:38:50 +0100 | [diff] [blame] | 1463 | config_info_line( |
| 1464 | 'mkl_aarch64', |
| 1465 | 'Build with oneDNN and Compute Library for the Arm Architecture (ACL).') |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1466 | config_info_line('monolithic', 'Config for mostly static monolithic build.') |
A. Unique TensorFlower | a6bf9c8 | 2019-02-26 10:08:35 -0800 | [diff] [blame] | 1467 | config_info_line('numa', 'Build with NUMA support.') |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 1468 | config_info_line( |
| 1469 | 'dynamic_kernels', |
| 1470 | '(Experimental) Build kernels into separate shared objects.') |
Ian Hua | 6b51112 | 2021-08-11 21:38:50 +0100 | [diff] [blame] | 1471 | config_info_line('v1', 'Build with TensorFlow 1 API instead of TF 2 API.') |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1472 | |
| 1473 | print('Preconfigured Bazel build configs to DISABLE default on features:') |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1474 | config_info_line('nogcp', 'Disable GCP support.') |
Gunhan Gulsoy | eea8168 | 2018-11-26 16:51:23 -0800 | [diff] [blame] | 1475 | config_info_line('nonccl', 'Disable NVIDIA NCCL support.') |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1476 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1477 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1478 | if __name__ == '__main__': |
| 1479 | main() |