A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | # ============================================================================== |
| 15 | """configure script to get build parameters from user.""" |
| 16 | |
| 17 | from __future__ import absolute_import |
| 18 | from __future__ import division |
| 19 | from __future__ import print_function |
| 20 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 21 | import argparse |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 22 | import errno |
| 23 | import os |
| 24 | import platform |
| 25 | import re |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 26 | import subprocess |
| 27 | import sys |
| 28 | |
Andrew Selle | c9885ea | 2017-11-06 09:37:03 -0800 | [diff] [blame] | 29 | # pylint: disable=g-import-not-at-top |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 30 | try: |
| 31 | from shutil import which |
| 32 | except ImportError: |
| 33 | from distutils.spawn import find_executable as which |
Andrew Selle | c9885ea | 2017-11-06 09:37:03 -0800 | [diff] [blame] | 34 | # pylint: enable=g-import-not-at-top |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 35 | |
Smit Hinsu | fe7d1d9 | 2018-07-14 13:16:58 -0700 | [diff] [blame] | 36 | _DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 37 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 38 | _TF_OPENCL_VERSION = '1.2' |
| 39 | _DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp' |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 40 | _DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include' |
A. Unique TensorFlower | 82820ef | 2018-11-12 13:22:13 -0800 | [diff] [blame] | 41 | _SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15, 16, 17, 18] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 42 | |
| 43 | _DEFAULT_PROMPT_ASK_ATTEMPTS = 10 |
| 44 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 45 | _TF_BAZELRC_FILENAME = '.tf_configure.bazelrc' |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 46 | _TF_WORKSPACE_ROOT = '' |
| 47 | _TF_BAZELRC = '' |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 48 | _TF_CURRENT_BAZEL_VERSION = None |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 49 | |
Jason Furmanek | 7c23415 | 2018-09-26 04:44:12 +0000 | [diff] [blame] | 50 | NCCL_LIB_PATHS = [ |
TensorFlower Gardener | ea5c529 | 2018-10-01 17:28:26 -0700 | [diff] [blame] | 51 | 'lib64/', 'lib/powerpc64le-linux-gnu/', 'lib/x86_64-linux-gnu/', '' |
Jason Furmanek | 7c23415 | 2018-09-26 04:44:12 +0000 | [diff] [blame] | 52 | ] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 53 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 54 | # List of files to configure when building Bazel on Apple platforms. |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 55 | APPLE_BAZEL_FILES = [ |
A. Unique TensorFlower | 6a059c0 | 2019-04-05 14:28:27 -0700 | [diff] [blame] | 56 | 'tensorflow/lite/experimental/ios/BUILD', |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 57 | 'tensorflow/lite/experimental/objc/BUILD', |
| 58 | 'tensorflow/lite/experimental/swift/BUILD' |
| 59 | ] |
| 60 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 61 | # List of files to move when building for iOS. |
| 62 | IOS_FILES = [ |
| 63 | 'tensorflow/lite/experimental/objc/TensorFlowLiteObjC.podspec', |
| 64 | 'tensorflow/lite/experimental/swift/TensorFlowLiteSwift.podspec', |
| 65 | ] |
| 66 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 67 | |
| 68 | class UserInputError(Exception): |
| 69 | pass |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 70 | |
| 71 | |
| 72 | def is_windows(): |
| 73 | return platform.system() == 'Windows' |
| 74 | |
| 75 | |
| 76 | def is_linux(): |
| 77 | return platform.system() == 'Linux' |
| 78 | |
| 79 | |
| 80 | def is_macos(): |
| 81 | return platform.system() == 'Darwin' |
| 82 | |
| 83 | |
| 84 | def is_ppc64le(): |
| 85 | return platform.machine() == 'ppc64le' |
| 86 | |
| 87 | |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 88 | def is_cygwin(): |
| 89 | return platform.system().startswith('CYGWIN_NT') |
| 90 | |
| 91 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 92 | def get_input(question): |
| 93 | try: |
| 94 | try: |
| 95 | answer = raw_input(question) |
| 96 | except NameError: |
| 97 | answer = input(question) # pylint: disable=bad-builtin |
| 98 | except EOFError: |
| 99 | answer = '' |
| 100 | return answer |
| 101 | |
| 102 | |
| 103 | def symlink_force(target, link_name): |
| 104 | """Force symlink, equivalent of 'ln -sf'. |
| 105 | |
| 106 | Args: |
| 107 | target: items to link to. |
| 108 | link_name: name of the link. |
| 109 | """ |
| 110 | try: |
| 111 | os.symlink(target, link_name) |
| 112 | except OSError as e: |
| 113 | if e.errno == errno.EEXIST: |
| 114 | os.remove(link_name) |
| 115 | os.symlink(target, link_name) |
| 116 | else: |
| 117 | raise e |
| 118 | |
| 119 | |
| 120 | def sed_in_place(filename, old, new): |
| 121 | """Replace old string with new string in file. |
| 122 | |
| 123 | Args: |
| 124 | filename: string for filename. |
| 125 | old: string to replace. |
| 126 | new: new string to replace to. |
| 127 | """ |
| 128 | with open(filename, 'r') as f: |
| 129 | filedata = f.read() |
| 130 | newdata = filedata.replace(old, new) |
| 131 | with open(filename, 'w') as f: |
| 132 | f.write(newdata) |
| 133 | |
| 134 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 135 | def write_to_bazelrc(line): |
| 136 | with open(_TF_BAZELRC, 'a') as f: |
| 137 | f.write(line + '\n') |
| 138 | |
| 139 | |
| 140 | def write_action_env_to_bazelrc(var_name, var): |
| 141 | write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var))) |
| 142 | |
| 143 | |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 144 | def run_shell(cmd, allow_non_zero=False): |
| 145 | if allow_non_zero: |
| 146 | try: |
| 147 | output = subprocess.check_output(cmd) |
| 148 | except subprocess.CalledProcessError as e: |
| 149 | output = e.output |
| 150 | else: |
| 151 | output = subprocess.check_output(cmd) |
| 152 | return output.decode('UTF-8').strip() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 153 | |
| 154 | |
| 155 | def cygpath(path): |
| 156 | """Convert path from posix to windows.""" |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 157 | return os.path.abspath(path).replace('\\', '/') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 158 | |
| 159 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 160 | def get_python_path(environ_cp, python_bin_path): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 161 | """Get the python site package paths.""" |
| 162 | python_paths = [] |
| 163 | if environ_cp.get('PYTHONPATH'): |
| 164 | python_paths = environ_cp.get('PYTHONPATH').split(':') |
| 165 | try: |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 166 | library_paths = run_shell([ |
| 167 | python_bin_path, '-c', |
| 168 | 'import site; print("\\n".join(site.getsitepackages()))' |
| 169 | ]).split('\n') |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 170 | except subprocess.CalledProcessError: |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 171 | library_paths = [ |
| 172 | run_shell([ |
| 173 | python_bin_path, '-c', |
| 174 | 'from distutils.sysconfig import get_python_lib;' |
| 175 | 'print(get_python_lib())' |
| 176 | ]) |
| 177 | ] |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 178 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 179 | all_paths = set(python_paths + library_paths) |
| 180 | |
| 181 | paths = [] |
| 182 | for path in all_paths: |
| 183 | if os.path.isdir(path): |
| 184 | paths.append(path) |
| 185 | return paths |
| 186 | |
| 187 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 188 | def get_python_major_version(python_bin_path): |
| 189 | """Get the python major version.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 190 | return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])']) |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 191 | |
| 192 | |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 193 | def setup_python(environ_cp): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 194 | """Setup python related env variables.""" |
| 195 | # Get PYTHON_BIN_PATH, default is the current running python. |
| 196 | default_python_bin_path = sys.executable |
| 197 | ask_python_bin_path = ('Please specify the location of python. [Default is ' |
| 198 | '%s]: ') % default_python_bin_path |
| 199 | while True: |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 200 | python_bin_path = get_from_env_or_user_or_default(environ_cp, |
| 201 | 'PYTHON_BIN_PATH', |
| 202 | ask_python_bin_path, |
| 203 | default_python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 204 | # Check if the path is valid |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 205 | if os.path.isfile(python_bin_path) and os.access(python_bin_path, os.X_OK): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 206 | break |
| 207 | elif not os.path.exists(python_bin_path): |
| 208 | print('Invalid python path: %s cannot be found.' % python_bin_path) |
| 209 | else: |
| 210 | print('%s is not executable. Is it the python binary?' % python_bin_path) |
| 211 | environ_cp['PYTHON_BIN_PATH'] = '' |
| 212 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 213 | # Convert python path to Windows style before checking lib and version |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 214 | if is_windows() or is_cygwin(): |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 215 | python_bin_path = cygpath(python_bin_path) |
| 216 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 217 | # Get PYTHON_LIB_PATH |
| 218 | python_lib_path = environ_cp.get('PYTHON_LIB_PATH') |
| 219 | if not python_lib_path: |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 220 | python_lib_paths = get_python_path(environ_cp, python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 221 | if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1': |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 222 | python_lib_path = python_lib_paths[0] |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 223 | else: |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 224 | print('Found possible Python library paths:\n %s' % |
| 225 | '\n '.join(python_lib_paths)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 226 | default_python_lib_path = python_lib_paths[0] |
| 227 | python_lib_path = get_input( |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 228 | 'Please input the desired Python library path to use. ' |
| 229 | 'Default is [%s]\n' % python_lib_paths[0]) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 230 | if not python_lib_path: |
| 231 | python_lib_path = default_python_lib_path |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 232 | environ_cp['PYTHON_LIB_PATH'] = python_lib_path |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 233 | |
TensorFlower Gardener | 61a8720 | 2018-10-01 12:25:39 -0700 | [diff] [blame] | 234 | _ = get_python_major_version(python_bin_path) |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 235 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 236 | # Convert python path to Windows style before writing into bazel.rc |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 237 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 238 | python_lib_path = cygpath(python_lib_path) |
| 239 | |
| 240 | # Set-up env variables used by python_configure.bzl |
| 241 | write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path) |
| 242 | write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path) |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 243 | write_to_bazelrc('build --python_path=\"%s"' % python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 244 | environ_cp['PYTHON_BIN_PATH'] = python_bin_path |
| 245 | |
William D. Irons | dcc76a5 | 2018-11-20 10:35:18 -0600 | [diff] [blame] | 246 | # If choosen python_lib_path is from a path specified in the PYTHONPATH |
| 247 | # variable, need to tell bazel to include PYTHONPATH |
| 248 | if environ_cp.get('PYTHONPATH'): |
| 249 | python_paths = environ_cp.get('PYTHONPATH').split(':') |
| 250 | if python_lib_path in python_paths: |
TensorFlower Gardener | 968cd18 | 2018-11-28 11:33:16 -0800 | [diff] [blame] | 251 | write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH')) |
William D. Irons | dcc76a5 | 2018-11-20 10:35:18 -0600 | [diff] [blame] | 252 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 253 | # Write tools/python_bin_path.sh |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 254 | with open( |
| 255 | os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'), |
| 256 | 'w') as f: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 257 | f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path) |
| 258 | |
| 259 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 260 | def reset_tf_configure_bazelrc(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 261 | """Reset file that contains customized config settings.""" |
| 262 | open(_TF_BAZELRC, 'w').close() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 263 | |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 264 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 265 | def cleanup_makefile(): |
| 266 | """Delete any leftover BUILD files from the Makefile build. |
| 267 | |
| 268 | These files could interfere with Bazel parsing. |
| 269 | """ |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 270 | makefile_download_dir = os.path.join(_TF_WORKSPACE_ROOT, 'tensorflow', |
| 271 | 'contrib', 'makefile', 'downloads') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 272 | if os.path.isdir(makefile_download_dir): |
| 273 | for root, _, filenames in os.walk(makefile_download_dir): |
| 274 | for f in filenames: |
| 275 | if f.endswith('BUILD'): |
| 276 | os.remove(os.path.join(root, f)) |
| 277 | |
| 278 | |
| 279 | def get_var(environ_cp, |
| 280 | var_name, |
| 281 | query_item, |
| 282 | enabled_by_default, |
| 283 | question=None, |
| 284 | yes_reply=None, |
| 285 | no_reply=None): |
| 286 | """Get boolean input from user. |
| 287 | |
| 288 | If var_name is not set in env, ask user to enable query_item or not. If the |
| 289 | response is empty, use the default. |
| 290 | |
| 291 | Args: |
| 292 | environ_cp: copy of the os.environ. |
| 293 | var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". |
| 294 | query_item: string for feature related to the variable, e.g. "Hadoop File |
| 295 | System". |
| 296 | enabled_by_default: boolean for default behavior. |
| 297 | question: optional string for how to ask for user input. |
Michael Case | d90054e | 2018-02-07 14:36:00 -0800 | [diff] [blame] | 298 | yes_reply: optional string for reply when feature is enabled. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 299 | no_reply: optional string for reply when feature is disabled. |
| 300 | |
| 301 | Returns: |
| 302 | boolean value of the variable. |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 303 | |
| 304 | Raises: |
| 305 | UserInputError: if an environment variable is set, but it cannot be |
| 306 | interpreted as a boolean indicator, assume that the user has made a |
| 307 | scripting error, and will continue to provide invalid input. |
| 308 | Raise the error to avoid infinitely looping. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 309 | """ |
| 310 | if not question: |
| 311 | question = 'Do you wish to build TensorFlow with %s support?' % query_item |
| 312 | if not yes_reply: |
| 313 | yes_reply = '%s support will be enabled for TensorFlow.' % query_item |
| 314 | if not no_reply: |
| 315 | no_reply = 'No %s' % yes_reply |
| 316 | |
| 317 | yes_reply += '\n' |
| 318 | no_reply += '\n' |
| 319 | |
| 320 | if enabled_by_default: |
| 321 | question += ' [Y/n]: ' |
| 322 | else: |
| 323 | question += ' [y/N]: ' |
| 324 | |
| 325 | var = environ_cp.get(var_name) |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 326 | if var is not None: |
| 327 | var_content = var.strip().lower() |
| 328 | true_strings = ('1', 't', 'true', 'y', 'yes') |
| 329 | false_strings = ('0', 'f', 'false', 'n', 'no') |
| 330 | if var_content in true_strings: |
| 331 | var = True |
| 332 | elif var_content in false_strings: |
| 333 | var = False |
| 334 | else: |
| 335 | raise UserInputError( |
| 336 | 'Environment variable %s must be set as a boolean indicator.\n' |
| 337 | 'The following are accepted as TRUE : %s.\n' |
| 338 | 'The following are accepted as FALSE: %s.\n' |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 339 | 'Current value is %s.' % |
| 340 | (var_name, ', '.join(true_strings), ', '.join(false_strings), var)) |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 341 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 342 | while var is None: |
| 343 | user_input_origin = get_input(question) |
| 344 | user_input = user_input_origin.strip().lower() |
| 345 | if user_input == 'y': |
| 346 | print(yes_reply) |
| 347 | var = True |
| 348 | elif user_input == 'n': |
| 349 | print(no_reply) |
| 350 | var = False |
| 351 | elif not user_input: |
| 352 | if enabled_by_default: |
| 353 | print(yes_reply) |
| 354 | var = True |
| 355 | else: |
| 356 | print(no_reply) |
| 357 | var = False |
| 358 | else: |
| 359 | print('Invalid selection: %s' % user_input_origin) |
| 360 | return var |
| 361 | |
| 362 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 363 | def set_build_var(environ_cp, |
| 364 | var_name, |
| 365 | query_item, |
| 366 | option_name, |
| 367 | enabled_by_default, |
| 368 | bazel_config_name=None): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 369 | """Set if query_item will be enabled for the build. |
| 370 | |
| 371 | Ask user if query_item will be enabled. Default is used if no input is given. |
| 372 | Set subprocess environment variable and write to .bazelrc if enabled. |
| 373 | |
| 374 | Args: |
| 375 | environ_cp: copy of the os.environ. |
| 376 | var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". |
| 377 | query_item: string for feature related to the variable, e.g. "Hadoop File |
| 378 | System". |
| 379 | option_name: string for option to define in .bazelrc. |
| 380 | enabled_by_default: boolean for default behavior. |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 381 | bazel_config_name: Name for Bazel --config argument to enable build feature. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 382 | """ |
| 383 | |
| 384 | var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default))) |
| 385 | environ_cp[var_name] = var |
| 386 | if var == '1': |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 387 | write_to_bazelrc('build:%s --define %s=true' % |
| 388 | (bazel_config_name, option_name)) |
Yifei Feng | ec451f5 | 2018-10-05 12:53:50 -0700 | [diff] [blame] | 389 | write_to_bazelrc('build --config=%s' % bazel_config_name) |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 390 | elif bazel_config_name is not None: |
| 391 | # TODO(mikecase): Migrate all users of configure.py to use --config Bazel |
| 392 | # options and not to set build configs through environment variables. |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 393 | write_to_bazelrc('build:%s --define %s=true' % |
| 394 | (bazel_config_name, option_name)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 395 | |
| 396 | |
| 397 | def set_action_env_var(environ_cp, |
| 398 | var_name, |
| 399 | query_item, |
| 400 | enabled_by_default, |
| 401 | question=None, |
| 402 | yes_reply=None, |
| 403 | no_reply=None): |
| 404 | """Set boolean action_env variable. |
| 405 | |
| 406 | Ask user if query_item will be enabled. Default is used if no input is given. |
| 407 | Set environment variable and write to .bazelrc. |
| 408 | |
| 409 | Args: |
| 410 | environ_cp: copy of the os.environ. |
| 411 | var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". |
| 412 | query_item: string for feature related to the variable, e.g. "Hadoop File |
| 413 | System". |
| 414 | enabled_by_default: boolean for default behavior. |
| 415 | question: optional string for how to ask for user input. |
Michael Case | d90054e | 2018-02-07 14:36:00 -0800 | [diff] [blame] | 416 | yes_reply: optional string for reply when feature is enabled. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 417 | no_reply: optional string for reply when feature is disabled. |
| 418 | """ |
| 419 | var = int( |
| 420 | get_var(environ_cp, var_name, query_item, enabled_by_default, question, |
| 421 | yes_reply, no_reply)) |
| 422 | |
| 423 | write_action_env_to_bazelrc(var_name, var) |
| 424 | environ_cp[var_name] = str(var) |
| 425 | |
| 426 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 427 | def convert_version_to_int(version): |
| 428 | """Convert a version number to a integer that can be used to compare. |
| 429 | |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 430 | Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The |
| 431 | 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored. |
| 432 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 433 | Args: |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 434 | version: a version to be converted |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 435 | |
| 436 | Returns: |
| 437 | An integer if converted successfully, otherwise return None. |
| 438 | """ |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 439 | version = version.split('-')[0] |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 440 | version_segments = version.split('.') |
Austin Anderson | 87ea41d | 2019-04-04 10:03:50 -0700 | [diff] [blame] | 441 | # Treat "0.24" as "0.24.0" |
| 442 | if len(version_segments) == 2: |
| 443 | version_segments.append('0') |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 444 | for seg in version_segments: |
| 445 | if not seg.isdigit(): |
| 446 | return None |
| 447 | |
| 448 | version_str = ''.join(['%03d' % int(seg) for seg in version_segments]) |
| 449 | return int(version_str) |
| 450 | |
| 451 | |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 452 | def check_bazel_version(min_version, max_version): |
| 453 | """Check installed bazel version is between min_version and max_version. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 454 | |
| 455 | Args: |
| 456 | min_version: string for minimum bazel version. |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 457 | max_version: string for maximum bazel version. |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 458 | |
| 459 | Returns: |
| 460 | The bazel version detected. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 461 | """ |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 462 | if which('bazel') is None: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 463 | print('Cannot find bazel. Please install bazel.') |
| 464 | sys.exit(0) |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 465 | curr_version = run_shell( |
| 466 | ['bazel', '--batch', '--bazelrc=/dev/null', 'version']) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 467 | |
| 468 | for line in curr_version.split('\n'): |
| 469 | if 'Build label: ' in line: |
| 470 | curr_version = line.split('Build label: ')[1] |
| 471 | break |
| 472 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 473 | min_version_int = convert_version_to_int(min_version) |
| 474 | curr_version_int = convert_version_to_int(curr_version) |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 475 | max_version_int = convert_version_to_int(max_version) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 476 | |
| 477 | # Check if current bazel version can be detected properly. |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 478 | if not curr_version_int: |
| 479 | print('WARNING: current bazel installation is not a release version.') |
| 480 | print('Make sure you are running at least bazel %s' % min_version) |
| 481 | return curr_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 482 | |
Michael Case | d94271a | 2017-08-22 17:26:52 -0700 | [diff] [blame] | 483 | print('You have bazel %s installed.' % curr_version) |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 484 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 485 | if curr_version_int < min_version_int: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 486 | print('Please upgrade your bazel installation to version %s or higher to ' |
| 487 | 'build TensorFlow!' % min_version) |
Jason Zaman | b41761c | 2018-10-14 11:28:53 +0800 | [diff] [blame] | 488 | sys.exit(1) |
TensorFlower Gardener | 78c246b | 2018-12-13 12:37:42 -0800 | [diff] [blame] | 489 | if (curr_version_int > max_version_int and |
| 490 | 'TF_IGNORE_MAX_BAZEL_VERSION' not in os.environ): |
Mihai Maruseac | e7a123f | 2018-11-30 13:33:25 -0800 | [diff] [blame] | 491 | print('Please downgrade your bazel installation to version %s or lower to ' |
Mihai Maruseac | e0963c4 | 2018-12-20 14:27:40 -0800 | [diff] [blame] | 492 | 'build TensorFlow! To downgrade: download the installer for the old ' |
| 493 | 'version (from https://github.com/bazelbuild/bazel/releases) then ' |
| 494 | 'run the installer.' % max_version) |
Jason Zaman | b41761c | 2018-10-14 11:28:53 +0800 | [diff] [blame] | 495 | sys.exit(1) |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 496 | return curr_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 497 | |
| 498 | |
| 499 | def set_cc_opt_flags(environ_cp): |
| 500 | """Set up architecture-dependent optimization flags. |
| 501 | |
| 502 | Also append CC optimization flags to bazel.rc.. |
| 503 | |
| 504 | Args: |
| 505 | environ_cp: copy of the os.environ. |
| 506 | """ |
| 507 | if is_ppc64le(): |
| 508 | # gcc on ppc64le does not support -march, use mcpu instead |
| 509 | default_cc_opt_flags = '-mcpu=native' |
A. Unique TensorFlower | 1bba94a | 2018-04-04 15:45:20 -0700 | [diff] [blame] | 510 | elif is_windows(): |
| 511 | default_cc_opt_flags = '/arch:AVX' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 512 | else: |
Justin Lebar | 9ef04f5 | 2018-10-10 18:52:45 -0700 | [diff] [blame] | 513 | default_cc_opt_flags = '-march=native -Wno-sign-compare' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 514 | question = ('Please specify optimization flags to use during compilation when' |
| 515 | ' bazel option "--config=opt" is specified [Default is %s]: ' |
| 516 | ) % default_cc_opt_flags |
| 517 | cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS', |
| 518 | question, default_cc_opt_flags) |
| 519 | for opt in cc_opt_flags.split(): |
Michael Case | 0017742 | 2017-11-10 13:14:03 -0800 | [diff] [blame] | 520 | write_to_bazelrc('build:opt --copt=%s' % opt) |
| 521 | # It should be safe on the same build host. |
A. Unique TensorFlower | 1bba94a | 2018-04-04 15:45:20 -0700 | [diff] [blame] | 522 | if not is_ppc64le() and not is_windows(): |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 523 | write_to_bazelrc('build:opt --host_copt=-march=native') |
Michael Case | bb3355d | 2017-11-09 08:46:31 -0800 | [diff] [blame] | 524 | write_to_bazelrc('build:opt --define with_default_optimizations=true') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 525 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 526 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 527 | def set_tf_cuda_clang(environ_cp): |
| 528 | """set TF_CUDA_CLANG action_env. |
| 529 | |
| 530 | Args: |
| 531 | environ_cp: copy of the os.environ. |
| 532 | """ |
| 533 | question = 'Do you want to use clang as CUDA compiler?' |
| 534 | yes_reply = 'Clang will be used as CUDA compiler.' |
| 535 | no_reply = 'nvcc will be used as CUDA compiler.' |
| 536 | set_action_env_var( |
| 537 | environ_cp, |
| 538 | 'TF_CUDA_CLANG', |
| 539 | None, |
| 540 | False, |
| 541 | question=question, |
| 542 | yes_reply=yes_reply, |
| 543 | no_reply=no_reply) |
| 544 | |
| 545 | |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 546 | def set_tf_download_clang(environ_cp): |
| 547 | """Set TF_DOWNLOAD_CLANG action_env.""" |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 548 | question = 'Do you wish to download a fresh release of clang? (Experimental)' |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 549 | yes_reply = 'Clang will be downloaded and used to compile tensorflow.' |
| 550 | no_reply = 'Clang will not be downloaded.' |
| 551 | set_action_env_var( |
| 552 | environ_cp, |
| 553 | 'TF_DOWNLOAD_CLANG', |
| 554 | None, |
| 555 | False, |
| 556 | question=question, |
| 557 | yes_reply=yes_reply, |
| 558 | no_reply=no_reply) |
| 559 | |
| 560 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 561 | def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var, |
| 562 | var_default): |
| 563 | """Get var_name either from env, or user or default. |
| 564 | |
| 565 | If var_name has been set as environment variable, use the preset value, else |
| 566 | ask for user input. If no input is provided, the default is used. |
| 567 | |
| 568 | Args: |
| 569 | environ_cp: copy of the os.environ. |
| 570 | var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". |
| 571 | ask_for_var: string for how to ask for user input. |
| 572 | var_default: default value string. |
| 573 | |
| 574 | Returns: |
| 575 | string value for var_name |
| 576 | """ |
| 577 | var = environ_cp.get(var_name) |
| 578 | if not var: |
| 579 | var = get_input(ask_for_var) |
Michael Case | d94271a | 2017-08-22 17:26:52 -0700 | [diff] [blame] | 580 | print('\n') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 581 | if not var: |
| 582 | var = var_default |
| 583 | return var |
| 584 | |
| 585 | |
| 586 | def set_clang_cuda_compiler_path(environ_cp): |
| 587 | """Set CLANG_CUDA_COMPILER_PATH.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 588 | default_clang_path = which('clang') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 589 | ask_clang_path = ('Please specify which clang should be used as device and ' |
| 590 | 'host compiler. [Default is %s]: ') % default_clang_path |
| 591 | |
| 592 | while True: |
| 593 | clang_cuda_compiler_path = get_from_env_or_user_or_default( |
| 594 | environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path, |
| 595 | default_clang_path) |
| 596 | if os.path.exists(clang_cuda_compiler_path): |
| 597 | break |
| 598 | |
| 599 | # Reset and retry |
| 600 | print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path) |
| 601 | environ_cp['CLANG_CUDA_COMPILER_PATH'] = '' |
| 602 | |
| 603 | # Set CLANG_CUDA_COMPILER_PATH |
| 604 | environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path |
| 605 | write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH', |
| 606 | clang_cuda_compiler_path) |
| 607 | |
| 608 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 609 | def prompt_loop_or_load_from_env(environ_cp, |
| 610 | var_name, |
| 611 | var_default, |
| 612 | ask_for_var, |
| 613 | check_success, |
| 614 | error_msg, |
| 615 | suppress_default_error=False, |
| 616 | n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS): |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 617 | """Loop over user prompts for an ENV param until receiving a valid response. |
| 618 | |
| 619 | For the env param var_name, read from the environment or verify user input |
| 620 | until receiving valid input. When done, set var_name in the environ_cp to its |
| 621 | new value. |
| 622 | |
| 623 | Args: |
| 624 | environ_cp: (Dict) copy of the os.environ. |
| 625 | var_name: (String) string for name of environment variable, e.g. "TF_MYVAR". |
| 626 | var_default: (String) default value string. |
| 627 | ask_for_var: (String) string for how to ask for user input. |
| 628 | check_success: (Function) function that takes one argument and returns a |
| 629 | boolean. Should return True if the value provided is considered valid. May |
| 630 | contain a complex error message if error_msg does not provide enough |
| 631 | information. In that case, set suppress_default_error to True. |
| 632 | error_msg: (String) String with one and only one '%s'. Formatted with each |
| 633 | invalid response upon check_success(input) failure. |
| 634 | suppress_default_error: (Bool) Suppress the above error message in favor of |
| 635 | one from the check_success function. |
| 636 | n_ask_attempts: (Integer) Number of times to query for valid input before |
| 637 | raising an error and quitting. |
| 638 | |
| 639 | Returns: |
| 640 | [String] The value of var_name after querying for input. |
| 641 | |
| 642 | Raises: |
| 643 | UserInputError: if a query has been attempted n_ask_attempts times without |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 644 | success, assume that the user has made a scripting error, and will |
| 645 | continue to provide invalid input. Raise the error to avoid infinitely |
| 646 | looping. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 647 | """ |
| 648 | default = environ_cp.get(var_name) or var_default |
| 649 | full_query = '%s [Default is %s]: ' % ( |
| 650 | ask_for_var, |
| 651 | default, |
| 652 | ) |
| 653 | |
| 654 | for _ in range(n_ask_attempts): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 655 | val = get_from_env_or_user_or_default(environ_cp, var_name, full_query, |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 656 | default) |
| 657 | if check_success(val): |
| 658 | break |
| 659 | if not suppress_default_error: |
| 660 | print(error_msg % val) |
| 661 | environ_cp[var_name] = '' |
| 662 | else: |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 663 | raise UserInputError('Invalid %s setting was provided %d times in a row. ' |
| 664 | 'Assuming to be a scripting mistake.' % |
| 665 | (var_name, n_ask_attempts)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 666 | |
| 667 | environ_cp[var_name] = val |
| 668 | return val |
| 669 | |
| 670 | |
| 671 | def create_android_ndk_rule(environ_cp): |
| 672 | """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule.""" |
| 673 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 674 | default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' % |
| 675 | environ_cp['APPDATA']) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 676 | elif is_macos(): |
| 677 | default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 678 | else: |
| 679 | default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 680 | |
| 681 | def valid_ndk_path(path): |
| 682 | return (os.path.exists(path) and |
| 683 | os.path.exists(os.path.join(path, 'source.properties'))) |
| 684 | |
| 685 | android_ndk_home_path = prompt_loop_or_load_from_env( |
| 686 | environ_cp, |
| 687 | var_name='ANDROID_NDK_HOME', |
| 688 | var_default=default_ndk_path, |
| 689 | ask_for_var='Please specify the home path of the Android NDK to use.', |
| 690 | check_success=valid_ndk_path, |
| 691 | error_msg=('The path %s or its child file "source.properties" ' |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 692 | 'does not exist.')) |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 693 | write_action_env_to_bazelrc('ANDROID_NDK_HOME', android_ndk_home_path) |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 694 | write_action_env_to_bazelrc( |
| 695 | 'ANDROID_NDK_API_LEVEL', |
| 696 | get_ndk_api_level(environ_cp, android_ndk_home_path)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 697 | |
| 698 | |
| 699 | def create_android_sdk_rule(environ_cp): |
| 700 | """Set Android variables and write Android SDK WORKSPACE rule.""" |
| 701 | if is_windows() or is_cygwin(): |
| 702 | default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA']) |
| 703 | elif is_macos(): |
Shashi Shekhar | c0ff0cc | 2018-07-17 09:00:24 -0700 | [diff] [blame] | 704 | default_sdk_path = '%s/library/Android/Sdk' % environ_cp['HOME'] |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 705 | else: |
| 706 | default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME'] |
| 707 | |
| 708 | def valid_sdk_path(path): |
| 709 | return (os.path.exists(path) and |
| 710 | os.path.exists(os.path.join(path, 'platforms')) and |
| 711 | os.path.exists(os.path.join(path, 'build-tools'))) |
| 712 | |
| 713 | android_sdk_home_path = prompt_loop_or_load_from_env( |
| 714 | environ_cp, |
| 715 | var_name='ANDROID_SDK_HOME', |
| 716 | var_default=default_sdk_path, |
| 717 | ask_for_var='Please specify the home path of the Android SDK to use.', |
| 718 | check_success=valid_sdk_path, |
| 719 | error_msg=('Either %s does not exist, or it does not contain the ' |
| 720 | 'subdirectories "platforms" and "build-tools".')) |
| 721 | |
| 722 | platforms = os.path.join(android_sdk_home_path, 'platforms') |
| 723 | api_levels = sorted(os.listdir(platforms)) |
| 724 | api_levels = [x.replace('android-', '') for x in api_levels] |
| 725 | |
| 726 | def valid_api_level(api_level): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 727 | return os.path.exists( |
| 728 | os.path.join(android_sdk_home_path, 'platforms', |
| 729 | 'android-' + api_level)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 730 | |
| 731 | android_api_level = prompt_loop_or_load_from_env( |
| 732 | environ_cp, |
| 733 | var_name='ANDROID_API_LEVEL', |
| 734 | var_default=api_levels[-1], |
| 735 | ask_for_var=('Please specify the Android SDK API level to use. ' |
| 736 | '[Available levels: %s]') % api_levels, |
| 737 | check_success=valid_api_level, |
| 738 | error_msg='Android-%s is not present in the SDK path.') |
| 739 | |
| 740 | build_tools = os.path.join(android_sdk_home_path, 'build-tools') |
| 741 | versions = sorted(os.listdir(build_tools)) |
| 742 | |
| 743 | def valid_build_tools(version): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 744 | return os.path.exists( |
| 745 | os.path.join(android_sdk_home_path, 'build-tools', version)) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 746 | |
| 747 | android_build_tools_version = prompt_loop_or_load_from_env( |
| 748 | environ_cp, |
| 749 | var_name='ANDROID_BUILD_TOOLS_VERSION', |
| 750 | var_default=versions[-1], |
| 751 | ask_for_var=('Please specify an Android build tools version to use. ' |
| 752 | '[Available versions: %s]') % versions, |
| 753 | check_success=valid_build_tools, |
| 754 | error_msg=('The selected SDK does not have build-tools version %s ' |
| 755 | 'available.')) |
| 756 | |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 757 | write_action_env_to_bazelrc('ANDROID_BUILD_TOOLS_VERSION', |
| 758 | android_build_tools_version) |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 759 | write_action_env_to_bazelrc('ANDROID_SDK_API_LEVEL', android_api_level) |
| 760 | write_action_env_to_bazelrc('ANDROID_SDK_HOME', android_sdk_home_path) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 761 | |
| 762 | |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 763 | def get_ndk_api_level(environ_cp, android_ndk_home_path): |
| 764 | """Gets the appropriate NDK API level to use for the provided Android NDK path.""" |
| 765 | |
| 766 | # First check to see if we're using a blessed version of the NDK. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 767 | properties_path = '%s/source.properties' % android_ndk_home_path |
| 768 | if is_windows() or is_cygwin(): |
| 769 | properties_path = cygpath(properties_path) |
| 770 | with open(properties_path, 'r') as f: |
| 771 | filedata = f.read() |
| 772 | |
| 773 | revision = re.search(r'Pkg.Revision = (\d+)', filedata) |
| 774 | if revision: |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 775 | ndk_version = revision.group(1) |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 776 | else: |
| 777 | raise Exception('Unable to parse NDK revision.') |
Jared Duke | a0104b7 | 2019-04-04 12:23:58 -0700 | [diff] [blame] | 778 | if int(ndk_version) not in _SUPPORTED_ANDROID_NDK_VERSIONS: |
| 779 | print('WARNING: The NDK version in %s is %s, which is not ' |
| 780 | 'supported by Bazel (officially supported versions: %s). Please use ' |
| 781 | 'another version. Compiling Android targets may result in confusing ' |
| 782 | 'errors.\n' % (android_ndk_home_path, ndk_version, |
| 783 | _SUPPORTED_ANDROID_NDK_VERSIONS)) |
| 784 | |
| 785 | # Now grab the NDK API level to use. Note that this is different from the |
| 786 | # SDK API level, as the NDK API level is effectively the *min* target SDK |
| 787 | # version. |
| 788 | platforms = os.path.join(android_ndk_home_path, 'platforms') |
| 789 | api_levels = sorted(os.listdir(platforms)) |
| 790 | api_levels = [ |
| 791 | x.replace('android-', '') for x in api_levels if 'android-' in x |
| 792 | ] |
| 793 | |
| 794 | def valid_api_level(api_level): |
| 795 | return os.path.exists( |
| 796 | os.path.join(android_ndk_home_path, 'platforms', |
| 797 | 'android-' + api_level)) |
| 798 | |
| 799 | android_ndk_api_level = prompt_loop_or_load_from_env( |
| 800 | environ_cp, |
| 801 | var_name='ANDROID_NDK_API_LEVEL', |
| 802 | var_default='18', # 18 is required for GPU acceleration. |
| 803 | ask_for_var=('Please specify the (min) Android NDK API level to use. ' |
| 804 | '[Available levels: %s]') % api_levels, |
| 805 | check_success=valid_api_level, |
| 806 | error_msg='Android-%s is not present in the NDK path.') |
| 807 | |
| 808 | return android_ndk_api_level |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 809 | |
| 810 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 811 | def set_gcc_host_compiler_path(environ_cp): |
| 812 | """Set GCC_HOST_COMPILER_PATH.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 813 | default_gcc_host_compiler_path = which('gcc') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 814 | cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH') |
| 815 | |
| 816 | if os.path.islink(cuda_bin_symlink): |
| 817 | # os.readlink is only available in linux |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 818 | default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 819 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 820 | gcc_host_compiler_path = prompt_loop_or_load_from_env( |
| 821 | environ_cp, |
| 822 | var_name='GCC_HOST_COMPILER_PATH', |
| 823 | var_default=default_gcc_host_compiler_path, |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 824 | ask_for_var='Please specify which gcc should be used by nvcc as the host compiler.', |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 825 | check_success=os.path.exists, |
| 826 | error_msg='Invalid gcc path. %s cannot be found.', |
| 827 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 828 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 829 | write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path) |
| 830 | |
| 831 | |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 832 | def reformat_version_sequence(version_str, sequence_count): |
| 833 | """Reformat the version string to have the given number of sequences. |
| 834 | |
| 835 | For example: |
| 836 | Given (7, 2) -> 7.0 |
| 837 | (7.0.1, 2) -> 7.0 |
| 838 | (5, 1) -> 5 |
| 839 | (5.0.3.2, 1) -> 5 |
| 840 | |
| 841 | Args: |
| 842 | version_str: String, the version string. |
| 843 | sequence_count: int, an integer. |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 844 | |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 845 | Returns: |
| 846 | string, reformatted version string. |
| 847 | """ |
| 848 | v = version_str.split('.') |
| 849 | if len(v) < sequence_count: |
| 850 | v = v + (['0'] * (sequence_count - len(v))) |
| 851 | |
| 852 | return '.'.join(v[:sequence_count]) |
| 853 | |
| 854 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 855 | def set_tf_cuda_paths(environ_cp): |
| 856 | """Set TF_CUDA_PATHS.""" |
| 857 | ask_cuda_paths = ( |
| 858 | 'Please specify the comma-separated list of base paths to look for CUDA ' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 859 | 'libraries and headers. [Leave empty to use the default]: ') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 860 | tf_cuda_paths = get_from_env_or_user_or_default(environ_cp, 'TF_CUDA_PATHS', |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 861 | ask_cuda_paths, '') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 862 | if tf_cuda_paths: |
| 863 | environ_cp['TF_CUDA_PATHS'] = tf_cuda_paths |
| 864 | |
| 865 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 866 | def set_tf_cuda_version(environ_cp): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 867 | """Set TF_CUDA_VERSION.""" |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 868 | ask_cuda_version = ('Please specify the CUDA SDK version you want to use. ' |
| 869 | '[Leave empty to accept any version]: ') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 870 | tf_cuda_version = get_from_env_or_user_or_default(environ_cp, |
| 871 | 'TF_CUDA_VERSION', |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 872 | ask_cuda_version, '') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 873 | environ_cp['TF_CUDA_VERSION'] = tf_cuda_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 874 | |
| 875 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 876 | def set_tf_cudnn_version(environ_cp): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 877 | """Set TF_CUDNN_VERSION.""" |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 878 | ask_cudnn_version = ('Please specify the cuDNN version you want to use. ' |
| 879 | '[Leave empty to accept any version]: ') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 880 | tf_cudnn_version = get_from_env_or_user_or_default(environ_cp, |
| 881 | 'TF_CUDNN_VERSION', |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 882 | ask_cudnn_version, '') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 883 | environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 884 | |
| 885 | |
Mingxing Tan | 1e7b0e4 | 2018-06-28 19:13:20 -0700 | [diff] [blame] | 886 | def is_cuda_compatible(lib, cuda_ver, cudnn_ver): |
| 887 | """Check compatibility between given library and cudnn/cudart libraries.""" |
| 888 | ldd_bin = which('ldd') or '/usr/bin/ldd' |
| 889 | ldd_out = run_shell([ldd_bin, lib], True) |
| 890 | ldd_out = ldd_out.split(os.linesep) |
| 891 | cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$') |
| 892 | cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$') |
| 893 | cudnn = None |
| 894 | cudart = None |
| 895 | cudnn_ok = True # assume no cudnn dependency by default |
| 896 | cuda_ok = True # assume no cuda dependency by default |
| 897 | for line in ldd_out: |
| 898 | if 'libcudnn.so' in line: |
| 899 | cudnn = cudnn_pattern.search(line) |
| 900 | cudnn_ok = False |
| 901 | elif 'libcudart.so' in line: |
| 902 | cudart = cuda_pattern.search(line) |
| 903 | cuda_ok = False |
| 904 | if cudnn and len(cudnn.group(1)): |
| 905 | cudnn = convert_version_to_int(cudnn.group(1)) |
| 906 | if cudart and len(cudart.group(1)): |
| 907 | cudart = convert_version_to_int(cudart.group(1)) |
| 908 | if cudnn is not None: |
| 909 | cudnn_ok = (cudnn == cudnn_ver) |
| 910 | if cudart is not None: |
| 911 | cuda_ok = (cudart == cuda_ver) |
| 912 | return cudnn_ok and cuda_ok |
| 913 | |
| 914 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 915 | def set_tf_tensorrt_version(environ_cp): |
| 916 | """Set TF_TENSORRT_VERSION.""" |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 917 | if not is_linux(): |
| 918 | raise ValueError('Currently TensorRT is only supported on Linux platform.') |
| 919 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 920 | if not int(environ_cp.get('TF_NEED_TENSORRT', False)): |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 921 | return |
| 922 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 923 | ask_tensorrt_version = ( |
| 924 | 'Please specify the TensorRT version you want to use. ' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 925 | '[Leave empty to accept any version]: ') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 926 | tf_tensorrt_version = get_from_env_or_user_or_default(environ_cp, |
| 927 | 'TF_TENSORRT_VERSION', |
| 928 | ask_tensorrt_version, |
| 929 | '') |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 930 | environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 931 | |
| 932 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 933 | def set_tf_nccl_version(environ_cp): |
| 934 | """Set TF_NCCL_VERSION.""" |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 935 | if not is_linux(): |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 936 | raise ValueError('Currently NCCL is only supported on Linux platform.') |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 937 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 938 | if 'TF_NCCL_VERSION' in environ_cp: |
| 939 | return |
| 940 | |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 941 | ask_nccl_version = ( |
A. Unique TensorFlower | 53faa31 | 2018-10-05 08:46:54 -0700 | [diff] [blame] | 942 | 'Please specify the locally installed NCCL version you want to use. ' |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 943 | '[Leave empty to use http://github.com/nvidia/nccl]: ') |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 944 | tf_nccl_version = get_from_env_or_user_or_default(environ_cp, |
| 945 | 'TF_NCCL_VERSION', |
| 946 | ask_nccl_version, '') |
A. Unique TensorFlower | 1fda764 | 2018-04-05 03:09:27 -0700 | [diff] [blame] | 947 | environ_cp['TF_NCCL_VERSION'] = tf_nccl_version |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 948 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 949 | def get_native_cuda_compute_capabilities(environ_cp): |
| 950 | """Get native cuda compute capabilities. |
| 951 | |
| 952 | Args: |
| 953 | environ_cp: copy of the os.environ. |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 954 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 955 | Returns: |
| 956 | string of native cuda compute capabilities, separated by comma. |
| 957 | """ |
| 958 | device_query_bin = os.path.join( |
| 959 | environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery') |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 960 | if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK): |
| 961 | try: |
| 962 | output = run_shell(device_query_bin).split('\n') |
| 963 | pattern = re.compile('[0-9]*\\.[0-9]*') |
| 964 | output = [pattern.search(x) for x in output if 'Capability' in x] |
| 965 | output = ','.join(x.group() for x in output if x is not None) |
| 966 | except subprocess.CalledProcessError: |
| 967 | output = '' |
| 968 | else: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 969 | output = '' |
| 970 | return output |
| 971 | |
| 972 | |
| 973 | def set_tf_cuda_compute_capabilities(environ_cp): |
| 974 | """Set TF_CUDA_COMPUTE_CAPABILITIES.""" |
| 975 | while True: |
| 976 | native_cuda_compute_capabilities = get_native_cuda_compute_capabilities( |
| 977 | environ_cp) |
| 978 | if not native_cuda_compute_capabilities: |
| 979 | default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES |
| 980 | else: |
| 981 | default_cuda_compute_capabilities = native_cuda_compute_capabilities |
| 982 | |
| 983 | ask_cuda_compute_capabilities = ( |
| 984 | 'Please specify a list of comma-separated ' |
P Sudeepam | 5209356 | 2019-02-17 17:34:01 +0530 | [diff] [blame] | 985 | 'CUDA compute capabilities you want to ' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 986 | 'build with.\nYou can find the compute ' |
| 987 | 'capability of your device at: ' |
| 988 | 'https://developer.nvidia.com/cuda-gpus.\nPlease' |
| 989 | ' note that each additional compute ' |
| 990 | 'capability significantly increases your ' |
P Sudeepam | 5209356 | 2019-02-17 17:34:01 +0530 | [diff] [blame] | 991 | 'build time and binary size, and that ' |
| 992 | 'TensorFlow only supports compute ' |
P Sudeepam | 765ceda | 2019-02-17 17:39:08 +0530 | [diff] [blame] | 993 | 'capabilities >= 3.5 [Default is: %s]: ' % |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 994 | default_cuda_compute_capabilities) |
| 995 | tf_cuda_compute_capabilities = get_from_env_or_user_or_default( |
| 996 | environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES', |
| 997 | ask_cuda_compute_capabilities, default_cuda_compute_capabilities) |
| 998 | # Check whether all capabilities from the input is valid |
| 999 | all_valid = True |
Maciej | d0f5bc1 | 2018-04-30 22:30:58 -0500 | [diff] [blame] | 1000 | # Remove all whitespace characters before splitting the string |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 1001 | # that users may insert by accident, as this will result in error |
Maciej | d0f5bc1 | 2018-04-30 22:30:58 -0500 | [diff] [blame] | 1002 | tf_cuda_compute_capabilities = ''.join(tf_cuda_compute_capabilities.split()) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1003 | for compute_capability in tf_cuda_compute_capabilities.split(','): |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1004 | m = re.match('[0-9]+.[0-9]+', compute_capability) |
| 1005 | if not m: |
Austin Anderson | 32202dc | 2019-02-19 10:46:27 -0800 | [diff] [blame] | 1006 | print('Invalid compute capability: %s' % compute_capability) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1007 | all_valid = False |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1008 | else: |
P Sudeepam | 5209356 | 2019-02-17 17:34:01 +0530 | [diff] [blame] | 1009 | ver = float(m.group(0)) |
A. Unique TensorFlower | 8dc2d0e | 2019-03-12 01:41:05 -0700 | [diff] [blame] | 1010 | if ver < 3.0: |
| 1011 | print('ERROR: TensorFlow only supports CUDA compute capabilities 3.0 ' |
Austin Anderson | 32202dc | 2019-02-19 10:46:27 -0800 | [diff] [blame] | 1012 | 'and higher. Please re-specify the list of compute ' |
| 1013 | 'capabilities excluding version %s.' % ver) |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1014 | all_valid = False |
A. Unique TensorFlower | 8dc2d0e | 2019-03-12 01:41:05 -0700 | [diff] [blame] | 1015 | if ver < 3.5: |
| 1016 | print('WARNING: XLA does not support CUDA compute capabilities ' |
| 1017 | 'lower than 3.5. Disable XLA when running on older GPUs.') |
P Sudeepam | 765ceda | 2019-02-17 17:39:08 +0530 | [diff] [blame] | 1018 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1019 | if all_valid: |
| 1020 | break |
| 1021 | |
| 1022 | # Reset and Retry |
| 1023 | environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = '' |
| 1024 | |
| 1025 | # Set TF_CUDA_COMPUTE_CAPABILITIES |
| 1026 | environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities |
| 1027 | write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES', |
| 1028 | tf_cuda_compute_capabilities) |
| 1029 | |
| 1030 | |
| 1031 | def set_other_cuda_vars(environ_cp): |
| 1032 | """Set other CUDA related variables.""" |
A. Unique TensorFlower | ab39198 | 2018-07-11 04:52:49 -0700 | [diff] [blame] | 1033 | # If CUDA is enabled, always use GPU during build and test. |
| 1034 | if environ_cp.get('TF_CUDA_CLANG') == '1': |
| 1035 | write_to_bazelrc('build --config=cuda_clang') |
| 1036 | write_to_bazelrc('test --config=cuda_clang') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1037 | else: |
A. Unique TensorFlower | ab39198 | 2018-07-11 04:52:49 -0700 | [diff] [blame] | 1038 | write_to_bazelrc('build --config=cuda') |
| 1039 | write_to_bazelrc('test --config=cuda') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1040 | |
| 1041 | |
| 1042 | def set_host_cxx_compiler(environ_cp): |
| 1043 | """Set HOST_CXX_COMPILER.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1044 | default_cxx_host_compiler = which('g++') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1045 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1046 | host_cxx_compiler = prompt_loop_or_load_from_env( |
| 1047 | environ_cp, |
| 1048 | var_name='HOST_CXX_COMPILER', |
| 1049 | var_default=default_cxx_host_compiler, |
| 1050 | ask_for_var=('Please specify which C++ compiler should be used as the ' |
| 1051 | 'host C++ compiler.'), |
| 1052 | check_success=os.path.exists, |
| 1053 | error_msg='Invalid C++ compiler path. %s cannot be found.', |
| 1054 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1055 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1056 | write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler) |
| 1057 | |
| 1058 | |
| 1059 | def set_host_c_compiler(environ_cp): |
| 1060 | """Set HOST_C_COMPILER.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1061 | default_c_host_compiler = which('gcc') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1062 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1063 | host_c_compiler = prompt_loop_or_load_from_env( |
| 1064 | environ_cp, |
| 1065 | var_name='HOST_C_COMPILER', |
| 1066 | var_default=default_c_host_compiler, |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1067 | ask_for_var=('Please specify which C compiler should be used as the host ' |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1068 | 'C compiler.'), |
| 1069 | check_success=os.path.exists, |
| 1070 | error_msg='Invalid C compiler path. %s cannot be found.', |
| 1071 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1072 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1073 | write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler) |
| 1074 | |
| 1075 | |
| 1076 | def set_computecpp_toolkit_path(environ_cp): |
| 1077 | """Set COMPUTECPP_TOOLKIT_PATH.""" |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1078 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1079 | def toolkit_exists(toolkit_path): |
| 1080 | """Check if a computecpp toolkit path is valid.""" |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1081 | if is_linux(): |
| 1082 | sycl_rt_lib_path = 'lib/libComputeCpp.so' |
| 1083 | else: |
| 1084 | sycl_rt_lib_path = '' |
| 1085 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1086 | sycl_rt_lib_path_full = os.path.join(toolkit_path, sycl_rt_lib_path) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1087 | exists = os.path.exists(sycl_rt_lib_path_full) |
| 1088 | if not exists: |
| 1089 | print('Invalid SYCL %s library path. %s cannot be found' % |
| 1090 | (_TF_OPENCL_VERSION, sycl_rt_lib_path_full)) |
| 1091 | return exists |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1092 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1093 | computecpp_toolkit_path = prompt_loop_or_load_from_env( |
| 1094 | environ_cp, |
| 1095 | var_name='COMPUTECPP_TOOLKIT_PATH', |
| 1096 | var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH, |
| 1097 | ask_for_var=( |
| 1098 | 'Please specify the location where ComputeCpp for SYCL %s is ' |
| 1099 | 'installed.' % _TF_OPENCL_VERSION), |
| 1100 | check_success=toolkit_exists, |
| 1101 | error_msg='Invalid SYCL compiler path. %s cannot be found.', |
| 1102 | suppress_default_error=True) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1103 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1104 | write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH', |
| 1105 | computecpp_toolkit_path) |
| 1106 | |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1107 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1108 | def set_trisycl_include_dir(environ_cp): |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1109 | """Set TRISYCL_INCLUDE_DIR.""" |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 1110 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1111 | ask_trisycl_include_dir = ('Please specify the location of the triSYCL ' |
| 1112 | 'include directory. (Use --config=sycl_trisycl ' |
| 1113 | 'when building with Bazel) ' |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1114 | '[Default is %s]: ') % ( |
| 1115 | _DEFAULT_TRISYCL_INCLUDE_DIR) |
Frank Chen | c4ef927 | 2018-01-10 11:36:52 -0800 | [diff] [blame] | 1116 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1117 | while True: |
| 1118 | trisycl_include_dir = get_from_env_or_user_or_default( |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1119 | environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir, |
| 1120 | _DEFAULT_TRISYCL_INCLUDE_DIR) |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1121 | if os.path.exists(trisycl_include_dir): |
| 1122 | break |
| 1123 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1124 | print('Invalid triSYCL include directory, %s cannot be found' % |
| 1125 | (trisycl_include_dir)) |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1126 | |
| 1127 | # Set TRISYCL_INCLUDE_DIR |
| 1128 | environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1129 | write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', trisycl_include_dir) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1130 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1131 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1132 | def set_mpi_home(environ_cp): |
| 1133 | """Set MPI_HOME.""" |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1134 | |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1135 | default_mpi_home = which('mpirun') or which('mpiexec') or '' |
| 1136 | default_mpi_home = os.path.dirname(os.path.dirname(default_mpi_home)) |
| 1137 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1138 | def valid_mpi_path(mpi_home): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1139 | exists = ( |
| 1140 | os.path.exists(os.path.join(mpi_home, 'include')) and |
TensorFlower Gardener | f7c861f | 2018-11-01 11:29:40 -0700 | [diff] [blame] | 1141 | (os.path.exists(os.path.join(mpi_home, 'lib')) or |
| 1142 | os.path.exists(os.path.join(mpi_home, 'lib64')) or |
| 1143 | os.path.exists(os.path.join(mpi_home, 'lib32')))) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1144 | if not exists: |
TensorFlower Gardener | f7c861f | 2018-11-01 11:29:40 -0700 | [diff] [blame] | 1145 | print( |
| 1146 | 'Invalid path to the MPI Toolkit. %s or %s or %s or %s cannot be found' |
| 1147 | % (os.path.join(mpi_home, 'include'), |
Christian Goll | ba95d09 | 2018-10-04 17:06:23 +0200 | [diff] [blame] | 1148 | os.path.exists(os.path.join(mpi_home, 'lib')), |
| 1149 | os.path.exists(os.path.join(mpi_home, 'lib64')), |
| 1150 | os.path.exists(os.path.join(mpi_home, 'lib32')))) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1151 | return exists |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1152 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1153 | _ = prompt_loop_or_load_from_env( |
| 1154 | environ_cp, |
| 1155 | var_name='MPI_HOME', |
| 1156 | var_default=default_mpi_home, |
| 1157 | ask_for_var='Please specify the MPI toolkit folder.', |
| 1158 | check_success=valid_mpi_path, |
| 1159 | error_msg='', |
| 1160 | suppress_default_error=True) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1161 | |
| 1162 | |
| 1163 | def set_other_mpi_vars(environ_cp): |
| 1164 | """Set other MPI related variables.""" |
| 1165 | # Link the MPI header files |
| 1166 | mpi_home = environ_cp.get('MPI_HOME') |
| 1167 | symlink_force('%s/include/mpi.h' % mpi_home, 'third_party/mpi/mpi.h') |
| 1168 | |
| 1169 | # Determine if we use OpenMPI or MVAPICH, these require different header files |
| 1170 | # to be included here to make bazel dependency checker happy |
| 1171 | if os.path.exists(os.path.join(mpi_home, 'include/mpi_portable_platform.h')): |
| 1172 | symlink_force( |
| 1173 | os.path.join(mpi_home, 'include/mpi_portable_platform.h'), |
| 1174 | 'third_party/mpi/mpi_portable_platform.h') |
| 1175 | # TODO(gunan): avoid editing files in configure |
| 1176 | sed_in_place('third_party/mpi/mpi.bzl', 'MPI_LIB_IS_OPENMPI=False', |
| 1177 | 'MPI_LIB_IS_OPENMPI=True') |
| 1178 | else: |
| 1179 | # MVAPICH / MPICH |
| 1180 | symlink_force( |
| 1181 | os.path.join(mpi_home, 'include/mpio.h'), 'third_party/mpi/mpio.h') |
| 1182 | symlink_force( |
| 1183 | os.path.join(mpi_home, 'include/mpicxx.h'), 'third_party/mpi/mpicxx.h') |
| 1184 | # TODO(gunan): avoid editing files in configure |
| 1185 | sed_in_place('third_party/mpi/mpi.bzl', 'MPI_LIB_IS_OPENMPI=True', |
| 1186 | 'MPI_LIB_IS_OPENMPI=False') |
| 1187 | |
| 1188 | if os.path.exists(os.path.join(mpi_home, 'lib/libmpi.so')): |
| 1189 | symlink_force( |
| 1190 | os.path.join(mpi_home, 'lib/libmpi.so'), 'third_party/mpi/libmpi.so') |
Christian Goll | ba95d09 | 2018-10-04 17:06:23 +0200 | [diff] [blame] | 1191 | elif os.path.exists(os.path.join(mpi_home, 'lib64/libmpi.so')): |
| 1192 | symlink_force( |
| 1193 | os.path.join(mpi_home, 'lib64/libmpi.so'), 'third_party/mpi/libmpi.so') |
| 1194 | elif os.path.exists(os.path.join(mpi_home, 'lib32/libmpi.so')): |
| 1195 | symlink_force( |
| 1196 | os.path.join(mpi_home, 'lib32/libmpi.so'), 'third_party/mpi/libmpi.so') |
| 1197 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1198 | else: |
TensorFlower Gardener | f7c861f | 2018-11-01 11:29:40 -0700 | [diff] [blame] | 1199 | raise ValueError( |
| 1200 | 'Cannot find the MPI library file in %s/lib or %s/lib64 or %s/lib32' % |
Mihai Maruseac | 91ebeec | 2019-01-29 17:07:38 -0800 | [diff] [blame] | 1201 | (mpi_home, mpi_home, mpi_home)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1202 | |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 1203 | |
A. Unique TensorFlower | e9797fd | 2019-03-22 11:02:40 -0700 | [diff] [blame] | 1204 | def system_specific_test_config(env): |
A. Unique TensorFlower | 7bd8637 | 2019-03-21 15:19:30 -0700 | [diff] [blame] | 1205 | """Add default build and test flags required for TF tests to bazelrc.""" |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1206 | write_to_bazelrc('test --flaky_test_attempts=3') |
| 1207 | write_to_bazelrc('test --test_size_filters=small,medium') |
| 1208 | write_to_bazelrc( |
| 1209 | 'test --test_tag_filters=-benchmark-test,-no_oss,-oss_serial') |
| 1210 | write_to_bazelrc('test --build_tag_filters=-benchmark-test,-no_oss') |
| 1211 | if is_windows(): |
Guangda Lai | bcd701a | 2019-03-12 21:04:51 -0700 | [diff] [blame] | 1212 | if env.get('TF_NEED_CUDA', None) == '1': |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1213 | write_to_bazelrc( |
| 1214 | 'test --test_tag_filters=-no_windows,-no_windows_gpu,-no_gpu') |
| 1215 | write_to_bazelrc( |
| 1216 | 'test --build_tag_filters=-no_windows,-no_windows_gpu,-no_gpu') |
| 1217 | else: |
| 1218 | write_to_bazelrc('test --test_tag_filters=-no_windows,-gpu') |
| 1219 | write_to_bazelrc('test --build_tag_filters=-no_windows,-gpu') |
| 1220 | elif is_macos(): |
| 1221 | write_to_bazelrc('test --test_tag_filters=-gpu,-nomac,-no_mac') |
| 1222 | write_to_bazelrc('test --build_tag_filters=-gpu,-nomac,-no_mac') |
| 1223 | elif is_linux(): |
Guangda Lai | bcd701a | 2019-03-12 21:04:51 -0700 | [diff] [blame] | 1224 | if env.get('TF_NEED_CUDA', None) == '1': |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1225 | write_to_bazelrc('test --test_tag_filters=-no_gpu') |
| 1226 | write_to_bazelrc('test --build_tag_filters=-no_gpu') |
| 1227 | write_to_bazelrc('test --test_env=LD_LIBRARY_PATH') |
| 1228 | else: |
| 1229 | write_to_bazelrc('test --test_tag_filters=-gpu') |
| 1230 | write_to_bazelrc('test --build_tag_filters=-gpu') |
| 1231 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1232 | |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1233 | def set_system_libs_flag(environ_cp): |
| 1234 | syslibs = environ_cp.get('TF_SYSTEM_LIBS', '') |
TensorFlower Gardener | 61a8720 | 2018-10-01 12:25:39 -0700 | [diff] [blame] | 1235 | if syslibs: |
Jason Zaman | 5fc39bd | 2018-09-16 01:38:55 +0800 | [diff] [blame] | 1236 | if ',' in syslibs: |
| 1237 | syslibs = ','.join(sorted(syslibs.split(','))) |
| 1238 | else: |
| 1239 | syslibs = ','.join(sorted(syslibs.split())) |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1240 | write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs) |
| 1241 | |
Jason Zaman | 5fc39bd | 2018-09-16 01:38:55 +0800 | [diff] [blame] | 1242 | if 'PREFIX' in environ_cp: |
| 1243 | write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX']) |
| 1244 | if 'LIBDIR' in environ_cp: |
| 1245 | write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR']) |
| 1246 | if 'INCLUDEDIR' in environ_cp: |
| 1247 | write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR']) |
| 1248 | |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1249 | |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1250 | def set_windows_build_flags(environ_cp): |
| 1251 | """Set Windows specific build options.""" |
| 1252 | # The non-monolithic build is not supported yet |
| 1253 | write_to_bazelrc('build --config monolithic') |
| 1254 | # Suppress warning messages |
| 1255 | write_to_bazelrc('build --copt=-w --host_copt=-w') |
Loo Rong Jie | 31f10d2 | 2019-02-02 10:03:20 +0800 | [diff] [blame] | 1256 | # Fix winsock2.h conflicts |
TensorFlower Gardener | 345cccf | 2019-02-28 15:22:59 -0800 | [diff] [blame] | 1257 | write_to_bazelrc( |
| 1258 | 'build --copt=-DWIN32_LEAN_AND_MEAN --host_copt=-DWIN32_LEAN_AND_MEAN') |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1259 | # Output more verbose information when something goes wrong |
| 1260 | write_to_bazelrc('build --verbose_failures') |
| 1261 | # The host and target platforms are the same in Windows build. So we don't |
| 1262 | # have to distinct them. This avoids building the same targets twice. |
| 1263 | write_to_bazelrc('build --distinct_host_configuration=false') |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1264 | |
| 1265 | if get_var( |
| 1266 | environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline', |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1267 | True, ('Would you like to override eigen strong inline for some C++ ' |
| 1268 | 'compilation to reduce the compilation time?'), |
| 1269 | 'Eigen strong inline overridden.', 'Not overriding eigen strong inline, ' |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1270 | 'some compilations could take more than 20 mins.'): |
| 1271 | # Due to a known MSVC compiler issue |
| 1272 | # https://github.com/tensorflow/tensorflow/issues/10521 |
| 1273 | # Overriding eigen strong inline speeds up the compiling of |
| 1274 | # conv_grad_ops_3d.cc and conv_ops_3d.cc by 20 minutes, |
| 1275 | # but this also hurts the performance. Let users decide what they want. |
| 1276 | write_to_bazelrc('build --define=override_eigen_strong_inline=true') |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1277 | |
A. Unique TensorFlower | 061c359 | 2017-11-13 14:21:04 -0800 | [diff] [blame] | 1278 | |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1279 | def config_info_line(name, help_text): |
| 1280 | """Helper function to print formatted help text for Bazel config options.""" |
| 1281 | print('\t--config=%-12s\t# %s' % (name, help_text)) |
| 1282 | |
| 1283 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1284 | def configure_ios(): |
| 1285 | """Configures TensorFlow for iOS builds. |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1286 | |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1287 | This function will only be executed if `is_macos()` is true. |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1288 | """ |
| 1289 | if not is_macos(): |
| 1290 | return |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 1291 | if _TF_CURRENT_BAZEL_VERSION is None or _TF_CURRENT_BAZEL_VERSION < 23000: |
| 1292 | print( |
| 1293 | 'Building Bazel rules on Apple platforms requires Bazel 0.23 or later.') |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1294 | for filepath in APPLE_BAZEL_FILES: |
| 1295 | existing_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath + '.apple') |
| 1296 | renamed_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath) |
| 1297 | symlink_force(existing_filepath, renamed_filepath) |
| 1298 | for filepath in IOS_FILES: |
| 1299 | filename = os.path.basename(filepath) |
| 1300 | new_filepath = os.path.join(_TF_WORKSPACE_ROOT, filename) |
| 1301 | symlink_force(filepath, new_filepath) |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1302 | |
| 1303 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 1304 | def validate_cuda_config(environ_cp): |
| 1305 | """Run find_cuda_config.py and return cuda_toolkit_path, or None.""" |
| 1306 | |
| 1307 | def maybe_encode_env(env): |
| 1308 | """Encodes unicode in env to str on Windows python 2.x.""" |
| 1309 | if not is_windows() or sys.version_info[0] != 2: |
| 1310 | return env |
| 1311 | for k, v in env.items(): |
| 1312 | if isinstance(k, unicode): |
| 1313 | k = k.encode('ascii') |
| 1314 | if isinstance(v, unicode): |
| 1315 | v = v.encode('ascii') |
| 1316 | env[k] = v |
| 1317 | return env |
| 1318 | |
| 1319 | cuda_libraries = ['cuda', 'cudnn'] |
| 1320 | if is_linux(): |
| 1321 | if 'TF_TENSORRT_VERSION' in environ_cp: # if env variable exists |
| 1322 | cuda_libraries.append('tensorrt') |
| 1323 | if environ_cp.get('TF_NCCL_VERSION', None): # if env variable not empty |
| 1324 | cuda_libraries.append('nccl') |
| 1325 | |
| 1326 | proc = subprocess.Popen( |
| 1327 | [environ_cp['PYTHON_BIN_PATH'], 'third_party/gpus/find_cuda_config.py'] + |
| 1328 | cuda_libraries, |
| 1329 | stdout=subprocess.PIPE, |
| 1330 | env=maybe_encode_env(environ_cp)) |
| 1331 | |
| 1332 | if proc.wait(): |
| 1333 | # Errors from find_cuda_config.py were sent to stderr. |
| 1334 | print('\n\nAsking for detailed CUDA configuration...\n') |
| 1335 | return False |
| 1336 | |
| 1337 | config = dict( |
| 1338 | tuple(line.decode('ascii').rstrip().split(': ')) for line in proc.stdout) |
| 1339 | |
| 1340 | print('Found CUDA %s in:' % config['cuda_version']) |
| 1341 | print(' %s' % config['cuda_library_dir']) |
| 1342 | print(' %s' % config['cuda_include_dir']) |
| 1343 | |
| 1344 | print('Found cuDNN %s in:' % config['cudnn_version']) |
| 1345 | print(' %s' % config['cudnn_library_dir']) |
| 1346 | print(' %s' % config['cudnn_include_dir']) |
| 1347 | |
| 1348 | if 'tensorrt_version' in config: |
| 1349 | print('Found TensorRT %s in:' % config['tensorrt_version']) |
| 1350 | print(' %s' % config['tensorrt_library_dir']) |
| 1351 | print(' %s' % config['tensorrt_include_dir']) |
| 1352 | |
| 1353 | if config.get('nccl_version', None): |
| 1354 | print('Found NCCL %s in:' % config['nccl_version']) |
| 1355 | print(' %s' % config['nccl_library_dir']) |
| 1356 | print(' %s' % config['nccl_include_dir']) |
| 1357 | |
| 1358 | print('\n') |
| 1359 | |
| 1360 | environ_cp['CUDA_TOOLKIT_PATH'] = config['cuda_toolkit_path'] |
| 1361 | return True |
| 1362 | |
| 1363 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1364 | def main(): |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1365 | global _TF_WORKSPACE_ROOT |
| 1366 | global _TF_BAZELRC |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 1367 | global _TF_CURRENT_BAZEL_VERSION |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1368 | |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1369 | parser = argparse.ArgumentParser() |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1370 | parser.add_argument( |
| 1371 | '--workspace', |
| 1372 | type=str, |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1373 | default=os.path.abspath(os.path.dirname(__file__)), |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1374 | help='The absolute path to your active Bazel workspace.') |
Shanqing Cai | 7144571 | 2018-03-12 19:33:52 -0700 | [diff] [blame] | 1375 | args = parser.parse_args() |
| 1376 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1377 | _TF_WORKSPACE_ROOT = args.workspace |
| 1378 | _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME) |
| 1379 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1380 | # Make a copy of os.environ to be clear when functions and getting and setting |
| 1381 | # environment variables. |
| 1382 | environ_cp = dict(os.environ) |
| 1383 | |
A. Unique TensorFlower | 2b9c299 | 2019-04-04 10:30:19 -0700 | [diff] [blame] | 1384 | current_bazel_version = check_bazel_version('0.22.0', '0.24.1') |
A. Unique TensorFlower | ed29734 | 2019-03-15 11:25:28 -0700 | [diff] [blame] | 1385 | _TF_CURRENT_BAZEL_VERSION = convert_version_to_int(current_bazel_version) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1386 | |
A. Unique TensorFlower | 7c5eb35 | 2018-10-01 07:15:23 -0700 | [diff] [blame] | 1387 | reset_tf_configure_bazelrc() |
Yun Peng | 03e63a2 | 2018-11-07 11:18:53 +0100 | [diff] [blame] | 1388 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1389 | cleanup_makefile() |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 1390 | setup_python(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1391 | |
| 1392 | if is_windows(): |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1393 | environ_cp['TF_NEED_OPENCL_SYCL'] = '0' |
| 1394 | environ_cp['TF_NEED_COMPUTECPP'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1395 | environ_cp['TF_NEED_OPENCL'] = '0' |
| 1396 | environ_cp['TF_CUDA_CLANG'] = '0' |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 1397 | environ_cp['TF_NEED_TENSORRT'] = '0' |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1398 | # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on |
| 1399 | # Windows. |
| 1400 | environ_cp['TF_DOWNLOAD_CLANG'] = '0' |
A. Unique TensorFlower | 6e97fb3 | 2018-07-16 14:07:29 -0700 | [diff] [blame] | 1401 | environ_cp['TF_NEED_MPI'] = '0' |
| 1402 | environ_cp['TF_SET_ANDROID_WORKSPACE'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1403 | |
| 1404 | if is_macos(): |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 1405 | environ_cp['TF_NEED_TENSORRT'] = '0' |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1406 | else: |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1407 | environ_cp['TF_CONFIGURE_IOS'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1408 | |
Jon Triebenbach | 6896a74 | 2018-06-27 13:29:53 -0500 | [diff] [blame] | 1409 | # The numpy package on ppc64le uses OpenBLAS which has multi-threading |
| 1410 | # issues that lead to incorrect answers. Set OMP_NUM_THREADS=1 at |
| 1411 | # runtime to allow the Tensorflow testcases which compare numpy |
| 1412 | # results to Tensorflow results to succeed. |
| 1413 | if is_ppc64le(): |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1414 | write_action_env_to_bazelrc('OMP_NUM_THREADS', 1) |
Jon Triebenbach | 6896a74 | 2018-06-27 13:29:53 -0500 | [diff] [blame] | 1415 | |
Grzegorz Pawelczak | ec82efd | 2018-10-09 15:03:46 +0100 | [diff] [blame] | 1416 | xla_enabled_by_default = is_linux() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1417 | set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support', |
Grzegorz Pawelczak | ec82efd | 2018-10-09 15:03:46 +0100 | [diff] [blame] | 1418 | xla_enabled_by_default, 'xla') |
Gunhan Gulsoy | 3da0dff | 2018-09-26 11:55:50 -0700 | [diff] [blame] | 1419 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1420 | set_action_env_var(environ_cp, 'TF_NEED_OPENCL_SYCL', 'OpenCL SYCL', False) |
| 1421 | if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1422 | set_host_cxx_compiler(environ_cp) |
| 1423 | set_host_c_compiler(environ_cp) |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1424 | set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True) |
| 1425 | if environ_cp.get('TF_NEED_COMPUTECPP') == '1': |
| 1426 | set_computecpp_toolkit_path(environ_cp) |
| 1427 | else: |
| 1428 | set_trisycl_include_dir(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1429 | |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1430 | set_action_env_var(environ_cp, 'TF_NEED_ROCM', 'ROCm', False) |
| 1431 | if (environ_cp.get('TF_NEED_ROCM') == '1' and |
TensorFlower Gardener | 62e6016 | 2018-09-27 10:22:55 -0700 | [diff] [blame] | 1432 | 'LD_LIBRARY_PATH' in environ_cp and |
| 1433 | environ_cp.get('LD_LIBRARY_PATH') != '1'): |
| 1434 | write_action_env_to_bazelrc('LD_LIBRARY_PATH', |
| 1435 | environ_cp.get('LD_LIBRARY_PATH')) |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1436 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1437 | set_action_env_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False) |
A. Unique TensorFlower | 24cbb2a | 2017-09-08 07:45:44 -0700 | [diff] [blame] | 1438 | if (environ_cp.get('TF_NEED_CUDA') == '1' and |
| 1439 | 'TF_CUDA_CONFIG_REPO' not in environ_cp): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 1440 | |
| 1441 | set_action_env_var(environ_cp, 'TF_NEED_TENSORRT', 'TensorRT', False) |
| 1442 | |
| 1443 | environ_save = dict(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1444 | for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 1445 | |
| 1446 | if validate_cuda_config(environ_cp): |
| 1447 | cuda_env_names = [ |
| 1448 | 'TF_CUDA_VERSION', 'TF_CUBLAS_VERSION', 'TF_CUDNN_VERSION', |
| 1449 | 'TF_TENSORRT_VERSION', 'TF_NCCL_VERSION', 'TF_CUDA_PATHS', |
| 1450 | 'CUDA_TOOLKIT_PATH' |
| 1451 | ] |
| 1452 | for name in cuda_env_names: |
| 1453 | if name in environ_cp: |
| 1454 | write_action_env_to_bazelrc(name, environ_cp[name]) |
| 1455 | break |
| 1456 | |
| 1457 | # Restore settings changed below if CUDA config could not be validated. |
| 1458 | environ_cp = dict(environ_save) |
| 1459 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1460 | set_tf_cuda_version(environ_cp) |
| 1461 | set_tf_cudnn_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1462 | if is_linux(): |
| 1463 | set_tf_tensorrt_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1464 | set_tf_nccl_version(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1465 | |
A. Unique TensorFlower | 651b0bc | 2019-04-16 06:13:43 -0700 | [diff] [blame^] | 1466 | set_tf_cuda_paths(environ_cp) |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1467 | |
A. Unique TensorFlower | 8e9ca66 | 2019-04-15 12:47:11 -0700 | [diff] [blame] | 1468 | else: |
| 1469 | raise UserInputError( |
| 1470 | 'Invalid CUDA setting were provided %d ' |
| 1471 | 'times in a row. Assuming to be a scripting mistake.' % |
| 1472 | _DEFAULT_PROMPT_ASK_ATTEMPTS) |
| 1473 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1474 | set_tf_cuda_compute_capabilities(environ_cp) |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 1475 | if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get( |
| 1476 | 'LD_LIBRARY_PATH') != '1': |
| 1477 | write_action_env_to_bazelrc('LD_LIBRARY_PATH', |
| 1478 | environ_cp.get('LD_LIBRARY_PATH')) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1479 | |
| 1480 | set_tf_cuda_clang(environ_cp) |
| 1481 | if environ_cp.get('TF_CUDA_CLANG') == '1': |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1482 | # Ask whether we should download the clang toolchain. |
| 1483 | set_tf_download_clang(environ_cp) |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 1484 | if environ_cp.get('TF_DOWNLOAD_CLANG') != '1': |
| 1485 | # Set up which clang we should use as the cuda / host compiler. |
| 1486 | set_clang_cuda_compiler_path(environ_cp) |
Ilya Biryukov | 1c3d02e | 2018-09-04 03:09:52 -0700 | [diff] [blame] | 1487 | else: |
| 1488 | # Use downloaded LLD for linking. |
| 1489 | write_to_bazelrc('build:cuda_clang --config=download_clang_use_lld') |
| 1490 | write_to_bazelrc('test:cuda_clang --config=download_clang_use_lld') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1491 | else: |
| 1492 | # Set up which gcc nvcc should use as the host compiler |
| 1493 | # No need to set this on Windows |
| 1494 | if not is_windows(): |
| 1495 | set_gcc_host_compiler_path(environ_cp) |
| 1496 | set_other_cuda_vars(environ_cp) |
Ilya Biryukov | 9e651e4 | 2018-03-22 05:33:42 -0700 | [diff] [blame] | 1497 | else: |
| 1498 | # CUDA not required. Ask whether we should download the clang toolchain and |
| 1499 | # use it for the CPU build. |
| 1500 | set_tf_download_clang(environ_cp) |
| 1501 | if environ_cp.get('TF_DOWNLOAD_CLANG') == '1': |
| 1502 | write_to_bazelrc('build --config=download_clang') |
| 1503 | write_to_bazelrc('test --config=download_clang') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1504 | |
Wen-Heng (Jack) Chung | 69d3b8f | 2018-06-22 23:09:43 -0500 | [diff] [blame] | 1505 | # SYCL / ROCm / CUDA are mutually exclusive. |
| 1506 | # At most 1 GPU platform can be configured. |
| 1507 | gpu_platform_count = 0 |
| 1508 | if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': |
| 1509 | gpu_platform_count += 1 |
| 1510 | if environ_cp.get('TF_NEED_ROCM') == '1': |
| 1511 | gpu_platform_count += 1 |
| 1512 | if environ_cp.get('TF_NEED_CUDA') == '1': |
| 1513 | gpu_platform_count += 1 |
| 1514 | if gpu_platform_count >= 2: |
| 1515 | raise UserInputError('SYCL / CUDA / ROCm are mututally exclusive. ' |
| 1516 | 'At most 1 GPU platform can be configured.') |
| 1517 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1518 | set_build_var(environ_cp, 'TF_NEED_MPI', 'MPI', 'with_mpi_support', False) |
| 1519 | if environ_cp.get('TF_NEED_MPI') == '1': |
| 1520 | set_mpi_home(environ_cp) |
| 1521 | set_other_mpi_vars(environ_cp) |
| 1522 | |
| 1523 | set_cc_opt_flags(environ_cp) |
Yifei Feng | 5198cb8 | 2018-08-17 13:53:06 -0700 | [diff] [blame] | 1524 | set_system_libs_flag(environ_cp) |
A. Unique TensorFlower | 1b21235 | 2018-07-19 13:48:50 -0700 | [diff] [blame] | 1525 | if is_windows(): |
| 1526 | set_windows_build_flags(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1527 | |
Anna R | a9a1d5a | 2018-09-14 12:44:31 -0700 | [diff] [blame] | 1528 | # Add a config option to build TensorFlow 2.0 API. |
| 1529 | write_to_bazelrc('build:v2 --define=tf_api_version=2') |
| 1530 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1531 | if get_var(environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', False, |
| 1532 | ('Would you like to interactively configure ./WORKSPACE for ' |
| 1533 | 'Android builds?'), 'Searching for NDK and SDK installations.', |
| 1534 | 'Not configuring the WORKSPACE for Android builds.'): |
Michael Case | 5105350 | 2018-06-05 17:47:19 -0700 | [diff] [blame] | 1535 | create_android_ndk_rule(environ_cp) |
| 1536 | create_android_sdk_rule(environ_cp) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1537 | |
A. Unique TensorFlower | e9797fd | 2019-03-22 11:02:40 -0700 | [diff] [blame] | 1538 | system_specific_test_config(os.environ) |
Gunhan Gulsoy | fff6696 | 2019-02-20 15:28:43 -0800 | [diff] [blame] | 1539 | |
A. Unique TensorFlower | 5001ac0 | 2019-04-03 11:59:08 -0700 | [diff] [blame] | 1540 | set_action_env_var(environ_cp, 'TF_CONFIGURE_IOS', 'iOS', False) |
| 1541 | if environ_cp.get('TF_CONFIGURE_IOS') == '1': |
A. Unique TensorFlower | 76e879d | 2019-03-22 00:25:36 -0700 | [diff] [blame] | 1542 | configure_ios() |
A. Unique TensorFlower | e9797fd | 2019-03-22 11:02:40 -0700 | [diff] [blame] | 1543 | else: |
| 1544 | # TODO(pcloudy): Remove BAZEL_USE_CPP_ONLY_TOOLCHAIN after Bazel is upgraded |
| 1545 | # to 0.24.0. |
| 1546 | # For working around https://github.com/bazelbuild/bazel/issues/7607 |
| 1547 | if is_macos(): |
| 1548 | write_to_bazelrc('build --action_env=BAZEL_USE_CPP_ONLY_TOOLCHAIN=1') |
A. Unique TensorFlower | 93e7073 | 2019-02-14 16:45:32 -0800 | [diff] [blame] | 1549 | |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1550 | print('Preconfigured Bazel build configs. You can use any of the below by ' |
| 1551 | 'adding "--config=<>" to your build command. See .bazelrc for more ' |
| 1552 | 'details.') |
| 1553 | config_info_line('mkl', 'Build with MKL support.') |
| 1554 | config_info_line('monolithic', 'Config for mostly static monolithic build.') |
| 1555 | config_info_line('gdr', 'Build with GDR support.') |
| 1556 | config_info_line('verbs', 'Build with libverbs support.') |
| 1557 | config_info_line('ngraph', 'Build with Intel nGraph support.') |
A. Unique TensorFlower | a6bf9c8 | 2019-02-26 10:08:35 -0800 | [diff] [blame] | 1558 | config_info_line('numa', 'Build with NUMA support.') |
TensorFlower Gardener | abb763b | 2019-02-06 16:23:03 -0800 | [diff] [blame] | 1559 | config_info_line( |
| 1560 | 'dynamic_kernels', |
| 1561 | '(Experimental) Build kernels into separate shared objects.') |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1562 | |
| 1563 | print('Preconfigured Bazel build configs to DISABLE default on features:') |
| 1564 | config_info_line('noaws', 'Disable AWS S3 filesystem support.') |
| 1565 | config_info_line('nogcp', 'Disable GCP support.') |
| 1566 | config_info_line('nohdfs', 'Disable HDFS support.') |
Penporn Koanantakool | 489f1dc | 2019-01-10 22:07:22 -0800 | [diff] [blame] | 1567 | config_info_line('noignite', 'Disable Apache Ignite support.') |
Gunhan Gulsoy | 3437098 | 2018-10-12 15:26:01 -0700 | [diff] [blame] | 1568 | config_info_line('nokafka', 'Disable Apache Kafka support.') |
Gunhan Gulsoy | eea8168 | 2018-11-26 16:51:23 -0800 | [diff] [blame] | 1569 | config_info_line('nonccl', 'Disable NVIDIA NCCL support.') |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1570 | |
Gunhan Gulsoy | ffa90fc | 2018-09-26 01:38:55 -0700 | [diff] [blame] | 1571 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1572 | if __name__ == '__main__': |
| 1573 | main() |