A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | # ============================================================================== |
| 15 | """configure script to get build parameters from user.""" |
| 16 | |
| 17 | from __future__ import absolute_import |
| 18 | from __future__ import division |
| 19 | from __future__ import print_function |
| 20 | |
| 21 | import errno |
| 22 | import os |
| 23 | import platform |
| 24 | import re |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 25 | import subprocess |
| 26 | import sys |
| 27 | |
Andrew Selle | c9885ea | 2017-11-06 09:37:03 -0800 | [diff] [blame] | 28 | # pylint: disable=g-import-not-at-top |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 29 | try: |
| 30 | from shutil import which |
| 31 | except ImportError: |
| 32 | from distutils.spawn import find_executable as which |
Andrew Selle | c9885ea | 2017-11-06 09:37:03 -0800 | [diff] [blame] | 33 | # pylint: enable=g-import-not-at-top |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 34 | |
Michael Case | fe2c8d8 | 2017-10-02 13:54:34 -0700 | [diff] [blame] | 35 | _TF_BAZELRC = os.path.join(os.path.dirname(os.path.abspath(__file__)), |
| 36 | '.tf_configure.bazelrc') |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 37 | _TF_WORKSPACE = os.path.join(os.path.dirname(os.path.abspath(__file__)), |
| 38 | 'WORKSPACE') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 39 | _DEFAULT_CUDA_VERSION = '8.0' |
| 40 | _DEFAULT_CUDNN_VERSION = '6' |
| 41 | _DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,5.2' |
| 42 | _DEFAULT_CUDA_PATH = '/usr/local/cuda' |
| 43 | _DEFAULT_CUDA_PATH_LINUX = '/opt/cuda' |
| 44 | _DEFAULT_CUDA_PATH_WIN = ('C:/Program Files/NVIDIA GPU Computing ' |
| 45 | 'Toolkit/CUDA/v%s' % _DEFAULT_CUDA_VERSION) |
| 46 | _TF_OPENCL_VERSION = '1.2' |
| 47 | _DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp' |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 48 | _DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include' |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 49 | _SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15] |
| 50 | |
| 51 | _DEFAULT_PROMPT_ASK_ATTEMPTS = 10 |
| 52 | |
| 53 | |
| 54 | class UserInputError(Exception): |
| 55 | pass |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 56 | |
| 57 | |
| 58 | def is_windows(): |
| 59 | return platform.system() == 'Windows' |
| 60 | |
| 61 | |
| 62 | def is_linux(): |
| 63 | return platform.system() == 'Linux' |
| 64 | |
| 65 | |
| 66 | def is_macos(): |
| 67 | return platform.system() == 'Darwin' |
| 68 | |
| 69 | |
| 70 | def is_ppc64le(): |
| 71 | return platform.machine() == 'ppc64le' |
| 72 | |
| 73 | |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 74 | def is_cygwin(): |
| 75 | return platform.system().startswith('CYGWIN_NT') |
| 76 | |
| 77 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 78 | def get_input(question): |
| 79 | try: |
| 80 | try: |
| 81 | answer = raw_input(question) |
| 82 | except NameError: |
| 83 | answer = input(question) # pylint: disable=bad-builtin |
| 84 | except EOFError: |
| 85 | answer = '' |
| 86 | return answer |
| 87 | |
| 88 | |
| 89 | def symlink_force(target, link_name): |
| 90 | """Force symlink, equivalent of 'ln -sf'. |
| 91 | |
| 92 | Args: |
| 93 | target: items to link to. |
| 94 | link_name: name of the link. |
| 95 | """ |
| 96 | try: |
| 97 | os.symlink(target, link_name) |
| 98 | except OSError as e: |
| 99 | if e.errno == errno.EEXIST: |
| 100 | os.remove(link_name) |
| 101 | os.symlink(target, link_name) |
| 102 | else: |
| 103 | raise e |
| 104 | |
| 105 | |
| 106 | def sed_in_place(filename, old, new): |
| 107 | """Replace old string with new string in file. |
| 108 | |
| 109 | Args: |
| 110 | filename: string for filename. |
| 111 | old: string to replace. |
| 112 | new: new string to replace to. |
| 113 | """ |
| 114 | with open(filename, 'r') as f: |
| 115 | filedata = f.read() |
| 116 | newdata = filedata.replace(old, new) |
| 117 | with open(filename, 'w') as f: |
| 118 | f.write(newdata) |
| 119 | |
| 120 | |
| 121 | def remove_line_with(filename, token): |
| 122 | """Remove lines that contain token from file. |
| 123 | |
| 124 | Args: |
| 125 | filename: string for filename. |
| 126 | token: string token to check if to remove a line from file or not. |
| 127 | """ |
| 128 | with open(filename, 'r') as f: |
| 129 | filedata = f.read() |
| 130 | |
| 131 | with open(filename, 'w') as f: |
| 132 | for line in filedata.strip().split('\n'): |
| 133 | if token not in line: |
| 134 | f.write(line + '\n') |
| 135 | |
| 136 | |
| 137 | def write_to_bazelrc(line): |
| 138 | with open(_TF_BAZELRC, 'a') as f: |
| 139 | f.write(line + '\n') |
| 140 | |
| 141 | |
| 142 | def write_action_env_to_bazelrc(var_name, var): |
| 143 | write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var))) |
| 144 | |
| 145 | |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 146 | def run_shell(cmd, allow_non_zero=False): |
| 147 | if allow_non_zero: |
| 148 | try: |
| 149 | output = subprocess.check_output(cmd) |
| 150 | except subprocess.CalledProcessError as e: |
| 151 | output = e.output |
| 152 | else: |
| 153 | output = subprocess.check_output(cmd) |
| 154 | return output.decode('UTF-8').strip() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 155 | |
| 156 | |
| 157 | def cygpath(path): |
| 158 | """Convert path from posix to windows.""" |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 159 | return os.path.abspath(path).replace('\\', '/') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 160 | |
| 161 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 162 | def get_python_path(environ_cp, python_bin_path): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 163 | """Get the python site package paths.""" |
| 164 | python_paths = [] |
| 165 | if environ_cp.get('PYTHONPATH'): |
| 166 | python_paths = environ_cp.get('PYTHONPATH').split(':') |
| 167 | try: |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 168 | library_paths = run_shell( |
| 169 | [python_bin_path, '-c', |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 170 | 'import site; print("\\n".join(site.getsitepackages()))']).split('\n') |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 171 | except subprocess.CalledProcessError: |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 172 | library_paths = [run_shell( |
| 173 | [python_bin_path, '-c', |
| 174 | 'from distutils.sysconfig import get_python_lib;' |
| 175 | 'print(get_python_lib())'])] |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 176 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 177 | all_paths = set(python_paths + library_paths) |
| 178 | |
| 179 | paths = [] |
| 180 | for path in all_paths: |
| 181 | if os.path.isdir(path): |
| 182 | paths.append(path) |
| 183 | return paths |
| 184 | |
| 185 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 186 | def get_python_major_version(python_bin_path): |
| 187 | """Get the python major version.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 188 | return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])']) |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 189 | |
| 190 | |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 191 | def setup_python(environ_cp): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 192 | """Setup python related env variables.""" |
| 193 | # Get PYTHON_BIN_PATH, default is the current running python. |
| 194 | default_python_bin_path = sys.executable |
| 195 | ask_python_bin_path = ('Please specify the location of python. [Default is ' |
| 196 | '%s]: ') % default_python_bin_path |
| 197 | while True: |
| 198 | python_bin_path = get_from_env_or_user_or_default( |
| 199 | environ_cp, 'PYTHON_BIN_PATH', ask_python_bin_path, |
| 200 | default_python_bin_path) |
| 201 | # Check if the path is valid |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 202 | if os.path.isfile(python_bin_path) and os.access( |
| 203 | python_bin_path, os.X_OK): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 204 | break |
| 205 | elif not os.path.exists(python_bin_path): |
| 206 | print('Invalid python path: %s cannot be found.' % python_bin_path) |
| 207 | else: |
| 208 | print('%s is not executable. Is it the python binary?' % python_bin_path) |
| 209 | environ_cp['PYTHON_BIN_PATH'] = '' |
| 210 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 211 | # Convert python path to Windows style before checking lib and version |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 212 | if is_windows() or is_cygwin(): |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 213 | python_bin_path = cygpath(python_bin_path) |
| 214 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 215 | # Get PYTHON_LIB_PATH |
| 216 | python_lib_path = environ_cp.get('PYTHON_LIB_PATH') |
| 217 | if not python_lib_path: |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 218 | python_lib_paths = get_python_path(environ_cp, python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 219 | if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1': |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 220 | python_lib_path = python_lib_paths[0] |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 221 | else: |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 222 | print('Found possible Python library paths:\n %s' % |
| 223 | '\n '.join(python_lib_paths)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 224 | default_python_lib_path = python_lib_paths[0] |
| 225 | python_lib_path = get_input( |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 226 | 'Please input the desired Python library path to use. ' |
| 227 | 'Default is [%s]\n' % python_lib_paths[0]) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 228 | if not python_lib_path: |
| 229 | python_lib_path = default_python_lib_path |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 230 | environ_cp['PYTHON_LIB_PATH'] = python_lib_path |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 231 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 232 | python_major_version = get_python_major_version(python_bin_path) |
| 233 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 234 | # Convert python path to Windows style before writing into bazel.rc |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 235 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 236 | python_lib_path = cygpath(python_lib_path) |
| 237 | |
| 238 | # Set-up env variables used by python_configure.bzl |
| 239 | write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path) |
| 240 | write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 241 | write_to_bazelrc('build --force_python=py%s' % python_major_version) |
| 242 | write_to_bazelrc('build --host_force_python=py%s' % python_major_version) |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 243 | write_to_bazelrc('build --python_path=\"%s"' % python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 244 | environ_cp['PYTHON_BIN_PATH'] = python_bin_path |
| 245 | |
| 246 | # Write tools/python_bin_path.sh |
| 247 | with open('tools/python_bin_path.sh', 'w') as f: |
| 248 | f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path) |
| 249 | |
| 250 | |
| 251 | def reset_tf_configure_bazelrc(): |
| 252 | """Reset file that contains customized config settings.""" |
| 253 | open(_TF_BAZELRC, 'w').close() |
| 254 | |
| 255 | home = os.path.expanduser('~') |
| 256 | if not os.path.exists('.bazelrc'): |
| 257 | if os.path.exists(os.path.join(home, '.bazelrc')): |
| 258 | with open('.bazelrc', 'a') as f: |
Shanqing Cai | e2e3a94 | 2017-09-25 19:35:53 -0700 | [diff] [blame] | 259 | f.write('import %s/.bazelrc\n' % home.replace('\\', '/')) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 260 | else: |
| 261 | open('.bazelrc', 'w').close() |
| 262 | |
| 263 | remove_line_with('.bazelrc', 'tf_configure') |
| 264 | with open('.bazelrc', 'a') as f: |
| 265 | f.write('import %workspace%/.tf_configure.bazelrc\n') |
| 266 | |
| 267 | |
| 268 | def run_gen_git_source(environ_cp): |
| 269 | """Run the gen_git_source to create links. |
| 270 | |
| 271 | The links are for bazel to track dependencies for git hash propagation. |
| 272 | |
| 273 | Args: |
| 274 | environ_cp: copy of the os.environ. |
| 275 | """ |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 276 | cmd = '"%s" tensorflow/tools/git/gen_git_source.py --configure %s' % ( |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 277 | environ_cp.get('PYTHON_BIN_PATH'), os.getcwd()) |
| 278 | os.system(cmd) |
| 279 | |
| 280 | |
| 281 | def cleanup_makefile(): |
| 282 | """Delete any leftover BUILD files from the Makefile build. |
| 283 | |
| 284 | These files could interfere with Bazel parsing. |
| 285 | """ |
| 286 | makefile_download_dir = 'tensorflow/contrib/makefile/downloads' |
| 287 | if os.path.isdir(makefile_download_dir): |
| 288 | for root, _, filenames in os.walk(makefile_download_dir): |
| 289 | for f in filenames: |
| 290 | if f.endswith('BUILD'): |
| 291 | os.remove(os.path.join(root, f)) |
| 292 | |
| 293 | |
| 294 | def get_var(environ_cp, |
| 295 | var_name, |
| 296 | query_item, |
| 297 | enabled_by_default, |
| 298 | question=None, |
| 299 | yes_reply=None, |
| 300 | no_reply=None): |
| 301 | """Get boolean input from user. |
| 302 | |
| 303 | If var_name is not set in env, ask user to enable query_item or not. If the |
| 304 | response is empty, use the default. |
| 305 | |
| 306 | Args: |
| 307 | environ_cp: copy of the os.environ. |
| 308 | var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". |
| 309 | query_item: string for feature related to the variable, e.g. "Hadoop File |
| 310 | System". |
| 311 | enabled_by_default: boolean for default behavior. |
| 312 | question: optional string for how to ask for user input. |
| 313 | yes_reply: optionanl string for reply when feature is enabled. |
| 314 | no_reply: optional string for reply when feature is disabled. |
| 315 | |
| 316 | Returns: |
| 317 | boolean value of the variable. |
| 318 | """ |
| 319 | if not question: |
| 320 | question = 'Do you wish to build TensorFlow with %s support?' % query_item |
| 321 | if not yes_reply: |
| 322 | yes_reply = '%s support will be enabled for TensorFlow.' % query_item |
| 323 | if not no_reply: |
| 324 | no_reply = 'No %s' % yes_reply |
| 325 | |
| 326 | yes_reply += '\n' |
| 327 | no_reply += '\n' |
| 328 | |
| 329 | if enabled_by_default: |
| 330 | question += ' [Y/n]: ' |
| 331 | else: |
| 332 | question += ' [y/N]: ' |
| 333 | |
| 334 | var = environ_cp.get(var_name) |
| 335 | while var is None: |
| 336 | user_input_origin = get_input(question) |
| 337 | user_input = user_input_origin.strip().lower() |
| 338 | if user_input == 'y': |
| 339 | print(yes_reply) |
| 340 | var = True |
| 341 | elif user_input == 'n': |
| 342 | print(no_reply) |
| 343 | var = False |
| 344 | elif not user_input: |
| 345 | if enabled_by_default: |
| 346 | print(yes_reply) |
| 347 | var = True |
| 348 | else: |
| 349 | print(no_reply) |
| 350 | var = False |
| 351 | else: |
| 352 | print('Invalid selection: %s' % user_input_origin) |
| 353 | return var |
| 354 | |
| 355 | |
| 356 | def set_build_var(environ_cp, var_name, query_item, option_name, |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 357 | enabled_by_default, bazel_config_name=None): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 358 | """Set if query_item will be enabled for the build. |
| 359 | |
| 360 | Ask user if query_item will be enabled. Default is used if no input is given. |
| 361 | Set subprocess environment variable and write to .bazelrc if enabled. |
| 362 | |
| 363 | Args: |
| 364 | environ_cp: copy of the os.environ. |
| 365 | var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". |
| 366 | query_item: string for feature related to the variable, e.g. "Hadoop File |
| 367 | System". |
| 368 | option_name: string for option to define in .bazelrc. |
| 369 | enabled_by_default: boolean for default behavior. |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 370 | bazel_config_name: Name for Bazel --config argument to enable build feature. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 371 | """ |
| 372 | |
| 373 | var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default))) |
| 374 | environ_cp[var_name] = var |
| 375 | if var == '1': |
| 376 | write_to_bazelrc('build --define %s=true' % option_name) |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 377 | elif bazel_config_name is not None: |
| 378 | # TODO(mikecase): Migrate all users of configure.py to use --config Bazel |
| 379 | # options and not to set build configs through environment variables. |
| 380 | write_to_bazelrc('build:%s --define %s=true' |
| 381 | % (bazel_config_name, option_name)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 382 | |
| 383 | |
| 384 | def set_action_env_var(environ_cp, |
| 385 | var_name, |
| 386 | query_item, |
| 387 | enabled_by_default, |
| 388 | question=None, |
| 389 | yes_reply=None, |
| 390 | no_reply=None): |
| 391 | """Set boolean action_env variable. |
| 392 | |
| 393 | Ask user if query_item will be enabled. Default is used if no input is given. |
| 394 | Set environment variable and write to .bazelrc. |
| 395 | |
| 396 | Args: |
| 397 | environ_cp: copy of the os.environ. |
| 398 | var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". |
| 399 | query_item: string for feature related to the variable, e.g. "Hadoop File |
| 400 | System". |
| 401 | enabled_by_default: boolean for default behavior. |
| 402 | question: optional string for how to ask for user input. |
| 403 | yes_reply: optionanl string for reply when feature is enabled. |
| 404 | no_reply: optional string for reply when feature is disabled. |
| 405 | """ |
| 406 | var = int( |
| 407 | get_var(environ_cp, var_name, query_item, enabled_by_default, question, |
| 408 | yes_reply, no_reply)) |
| 409 | |
| 410 | write_action_env_to_bazelrc(var_name, var) |
| 411 | environ_cp[var_name] = str(var) |
| 412 | |
| 413 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 414 | def convert_version_to_int(version): |
| 415 | """Convert a version number to a integer that can be used to compare. |
| 416 | |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 417 | Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The |
| 418 | 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored. |
| 419 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 420 | Args: |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 421 | version: a version to be converted |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 422 | |
| 423 | Returns: |
| 424 | An integer if converted successfully, otherwise return None. |
| 425 | """ |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 426 | version = version.split('-')[0] |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 427 | version_segments = version.split('.') |
| 428 | for seg in version_segments: |
| 429 | if not seg.isdigit(): |
| 430 | return None |
| 431 | |
| 432 | version_str = ''.join(['%03d' % int(seg) for seg in version_segments]) |
| 433 | return int(version_str) |
| 434 | |
| 435 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 436 | def check_bazel_version(min_version): |
| 437 | """Check installed bezel version is at least min_version. |
| 438 | |
| 439 | Args: |
| 440 | min_version: string for minimum bazel version. |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 441 | |
| 442 | Returns: |
| 443 | The bazel version detected. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 444 | """ |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 445 | if which('bazel') is None: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 446 | print('Cannot find bazel. Please install bazel.') |
| 447 | sys.exit(0) |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 448 | curr_version = run_shell(['bazel', '--batch', 'version']) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 449 | |
| 450 | for line in curr_version.split('\n'): |
| 451 | if 'Build label: ' in line: |
| 452 | curr_version = line.split('Build label: ')[1] |
| 453 | break |
| 454 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 455 | min_version_int = convert_version_to_int(min_version) |
| 456 | curr_version_int = convert_version_to_int(curr_version) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 457 | |
| 458 | # Check if current bazel version can be detected properly. |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 459 | if not curr_version_int: |
| 460 | print('WARNING: current bazel installation is not a release version.') |
| 461 | print('Make sure you are running at least bazel %s' % min_version) |
| 462 | return curr_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 463 | |
Michael Case | d94271a | 2017-08-22 17:26:52 -0700 | [diff] [blame] | 464 | print('You have bazel %s installed.' % curr_version) |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 465 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 466 | if curr_version_int < min_version_int: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 467 | print('Please upgrade your bazel installation to version %s or higher to ' |
| 468 | 'build TensorFlow!' % min_version) |
| 469 | sys.exit(0) |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 470 | return curr_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 471 | |
| 472 | |
| 473 | def set_cc_opt_flags(environ_cp): |
| 474 | """Set up architecture-dependent optimization flags. |
| 475 | |
| 476 | Also append CC optimization flags to bazel.rc.. |
| 477 | |
| 478 | Args: |
| 479 | environ_cp: copy of the os.environ. |
| 480 | """ |
| 481 | if is_ppc64le(): |
| 482 | # gcc on ppc64le does not support -march, use mcpu instead |
| 483 | default_cc_opt_flags = '-mcpu=native' |
| 484 | else: |
| 485 | default_cc_opt_flags = '-march=native' |
| 486 | question = ('Please specify optimization flags to use during compilation when' |
| 487 | ' bazel option "--config=opt" is specified [Default is %s]: ' |
| 488 | ) % default_cc_opt_flags |
| 489 | cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS', |
| 490 | question, default_cc_opt_flags) |
| 491 | for opt in cc_opt_flags.split(): |
Michael Case | 0017742 | 2017-11-10 13:14:03 -0800 | [diff] [blame] | 492 | write_to_bazelrc('build:opt --copt=%s' % opt) |
| 493 | # It should be safe on the same build host. |
| 494 | write_to_bazelrc('build:opt --host_copt=-march=native') |
Michael Case | bb3355d | 2017-11-09 08:46:31 -0800 | [diff] [blame] | 495 | write_to_bazelrc('build:opt --define with_default_optimizations=true') |
Michael Case | 0017742 | 2017-11-10 13:14:03 -0800 | [diff] [blame] | 496 | # TODO(mikecase): Remove these default defines once we are able to get |
| 497 | # TF Lite targets building without them. |
Andrew Selle | 0b15439 | 2017-11-10 10:35:35 -0800 | [diff] [blame] | 498 | write_to_bazelrc('build --copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK') |
| 499 | write_to_bazelrc('build --host_copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 500 | |
| 501 | |
| 502 | def set_tf_cuda_clang(environ_cp): |
| 503 | """set TF_CUDA_CLANG action_env. |
| 504 | |
| 505 | Args: |
| 506 | environ_cp: copy of the os.environ. |
| 507 | """ |
| 508 | question = 'Do you want to use clang as CUDA compiler?' |
| 509 | yes_reply = 'Clang will be used as CUDA compiler.' |
| 510 | no_reply = 'nvcc will be used as CUDA compiler.' |
| 511 | set_action_env_var( |
| 512 | environ_cp, |
| 513 | 'TF_CUDA_CLANG', |
| 514 | None, |
| 515 | False, |
| 516 | question=question, |
| 517 | yes_reply=yes_reply, |
| 518 | no_reply=no_reply) |
| 519 | |
| 520 | |
| 521 | def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var, |
| 522 | var_default): |
| 523 | """Get var_name either from env, or user or default. |
| 524 | |
| 525 | If var_name has been set as environment variable, use the preset value, else |
| 526 | ask for user input. If no input is provided, the default is used. |
| 527 | |
| 528 | Args: |
| 529 | environ_cp: copy of the os.environ. |
| 530 | var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". |
| 531 | ask_for_var: string for how to ask for user input. |
| 532 | var_default: default value string. |
| 533 | |
| 534 | Returns: |
| 535 | string value for var_name |
| 536 | """ |
| 537 | var = environ_cp.get(var_name) |
| 538 | if not var: |
| 539 | var = get_input(ask_for_var) |
Michael Case | d94271a | 2017-08-22 17:26:52 -0700 | [diff] [blame] | 540 | print('\n') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 541 | if not var: |
| 542 | var = var_default |
| 543 | return var |
| 544 | |
| 545 | |
| 546 | def set_clang_cuda_compiler_path(environ_cp): |
| 547 | """Set CLANG_CUDA_COMPILER_PATH.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 548 | default_clang_path = which('clang') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 549 | ask_clang_path = ('Please specify which clang should be used as device and ' |
| 550 | 'host compiler. [Default is %s]: ') % default_clang_path |
| 551 | |
| 552 | while True: |
| 553 | clang_cuda_compiler_path = get_from_env_or_user_or_default( |
| 554 | environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path, |
| 555 | default_clang_path) |
| 556 | if os.path.exists(clang_cuda_compiler_path): |
| 557 | break |
| 558 | |
| 559 | # Reset and retry |
| 560 | print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path) |
| 561 | environ_cp['CLANG_CUDA_COMPILER_PATH'] = '' |
| 562 | |
| 563 | # Set CLANG_CUDA_COMPILER_PATH |
| 564 | environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path |
| 565 | write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH', |
| 566 | clang_cuda_compiler_path) |
| 567 | |
| 568 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 569 | def prompt_loop_or_load_from_env( |
| 570 | environ_cp, |
| 571 | var_name, |
| 572 | var_default, |
| 573 | ask_for_var, |
| 574 | check_success, |
| 575 | error_msg, |
| 576 | suppress_default_error=False, |
| 577 | n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS |
| 578 | ): |
| 579 | """Loop over user prompts for an ENV param until receiving a valid response. |
| 580 | |
| 581 | For the env param var_name, read from the environment or verify user input |
| 582 | until receiving valid input. When done, set var_name in the environ_cp to its |
| 583 | new value. |
| 584 | |
| 585 | Args: |
| 586 | environ_cp: (Dict) copy of the os.environ. |
| 587 | var_name: (String) string for name of environment variable, e.g. "TF_MYVAR". |
| 588 | var_default: (String) default value string. |
| 589 | ask_for_var: (String) string for how to ask for user input. |
| 590 | check_success: (Function) function that takes one argument and returns a |
| 591 | boolean. Should return True if the value provided is considered valid. May |
| 592 | contain a complex error message if error_msg does not provide enough |
| 593 | information. In that case, set suppress_default_error to True. |
| 594 | error_msg: (String) String with one and only one '%s'. Formatted with each |
| 595 | invalid response upon check_success(input) failure. |
| 596 | suppress_default_error: (Bool) Suppress the above error message in favor of |
| 597 | one from the check_success function. |
| 598 | n_ask_attempts: (Integer) Number of times to query for valid input before |
| 599 | raising an error and quitting. |
| 600 | |
| 601 | Returns: |
| 602 | [String] The value of var_name after querying for input. |
| 603 | |
| 604 | Raises: |
| 605 | UserInputError: if a query has been attempted n_ask_attempts times without |
| 606 | success, assume that the user has made a scripting error, and will continue |
| 607 | to provide invalid input. Raise the error to avoid infinitely looping. |
| 608 | """ |
| 609 | default = environ_cp.get(var_name) or var_default |
| 610 | full_query = '%s [Default is %s]: ' % ( |
| 611 | ask_for_var, |
| 612 | default, |
| 613 | ) |
| 614 | |
| 615 | for _ in range(n_ask_attempts): |
| 616 | val = get_from_env_or_user_or_default(environ_cp, |
| 617 | var_name, |
| 618 | full_query, |
| 619 | default) |
| 620 | if check_success(val): |
| 621 | break |
| 622 | if not suppress_default_error: |
| 623 | print(error_msg % val) |
| 624 | environ_cp[var_name] = '' |
| 625 | else: |
| 626 | raise UserInputError('Invalid %s setting was provided %d times in a row. ' |
| 627 | 'Assuming to be a scripting mistake.' % |
| 628 | (var_name, n_ask_attempts)) |
| 629 | |
| 630 | environ_cp[var_name] = val |
| 631 | return val |
| 632 | |
| 633 | |
| 634 | def create_android_ndk_rule(environ_cp): |
| 635 | """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule.""" |
| 636 | if is_windows() or is_cygwin(): |
| 637 | default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' % |
| 638 | environ_cp['APPDATA']) |
| 639 | elif is_macos(): |
| 640 | default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 641 | else: |
| 642 | default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 643 | |
| 644 | def valid_ndk_path(path): |
| 645 | return (os.path.exists(path) and |
| 646 | os.path.exists(os.path.join(path, 'source.properties'))) |
| 647 | |
| 648 | android_ndk_home_path = prompt_loop_or_load_from_env( |
| 649 | environ_cp, |
| 650 | var_name='ANDROID_NDK_HOME', |
| 651 | var_default=default_ndk_path, |
| 652 | ask_for_var='Please specify the home path of the Android NDK to use.', |
| 653 | check_success=valid_ndk_path, |
| 654 | error_msg=('The path %s or its child file "source.properties" ' |
| 655 | 'does not exist.') |
| 656 | ) |
| 657 | |
| 658 | write_android_ndk_workspace_rule(android_ndk_home_path) |
| 659 | |
| 660 | |
| 661 | def create_android_sdk_rule(environ_cp): |
| 662 | """Set Android variables and write Android SDK WORKSPACE rule.""" |
| 663 | if is_windows() or is_cygwin(): |
| 664 | default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA']) |
| 665 | elif is_macos(): |
| 666 | default_sdk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 667 | else: |
| 668 | default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME'] |
| 669 | |
| 670 | def valid_sdk_path(path): |
| 671 | return (os.path.exists(path) and |
| 672 | os.path.exists(os.path.join(path, 'platforms')) and |
| 673 | os.path.exists(os.path.join(path, 'build-tools'))) |
| 674 | |
| 675 | android_sdk_home_path = prompt_loop_or_load_from_env( |
| 676 | environ_cp, |
| 677 | var_name='ANDROID_SDK_HOME', |
| 678 | var_default=default_sdk_path, |
| 679 | ask_for_var='Please specify the home path of the Android SDK to use.', |
| 680 | check_success=valid_sdk_path, |
| 681 | error_msg=('Either %s does not exist, or it does not contain the ' |
| 682 | 'subdirectories "platforms" and "build-tools".')) |
| 683 | |
| 684 | platforms = os.path.join(android_sdk_home_path, 'platforms') |
| 685 | api_levels = sorted(os.listdir(platforms)) |
| 686 | api_levels = [x.replace('android-', '') for x in api_levels] |
| 687 | |
| 688 | def valid_api_level(api_level): |
| 689 | return os.path.exists(os.path.join(android_sdk_home_path, |
| 690 | 'platforms', |
| 691 | 'android-' + api_level)) |
| 692 | |
| 693 | android_api_level = prompt_loop_or_load_from_env( |
| 694 | environ_cp, |
| 695 | var_name='ANDROID_API_LEVEL', |
| 696 | var_default=api_levels[-1], |
| 697 | ask_for_var=('Please specify the Android SDK API level to use. ' |
| 698 | '[Available levels: %s]') % api_levels, |
| 699 | check_success=valid_api_level, |
| 700 | error_msg='Android-%s is not present in the SDK path.') |
| 701 | |
| 702 | build_tools = os.path.join(android_sdk_home_path, 'build-tools') |
| 703 | versions = sorted(os.listdir(build_tools)) |
| 704 | |
| 705 | def valid_build_tools(version): |
| 706 | return os.path.exists(os.path.join(android_sdk_home_path, |
| 707 | 'build-tools', |
| 708 | version)) |
| 709 | |
| 710 | android_build_tools_version = prompt_loop_or_load_from_env( |
| 711 | environ_cp, |
| 712 | var_name='ANDROID_BUILD_TOOLS_VERSION', |
| 713 | var_default=versions[-1], |
| 714 | ask_for_var=('Please specify an Android build tools version to use. ' |
| 715 | '[Available versions: %s]') % versions, |
| 716 | check_success=valid_build_tools, |
| 717 | error_msg=('The selected SDK does not have build-tools version %s ' |
| 718 | 'available.')) |
| 719 | |
| 720 | write_android_sdk_workspace_rule(android_sdk_home_path, |
| 721 | android_build_tools_version, |
| 722 | android_api_level) |
| 723 | |
| 724 | |
| 725 | def write_android_sdk_workspace_rule(android_sdk_home_path, |
| 726 | android_build_tools_version, |
| 727 | android_api_level): |
| 728 | print('Writing android_sdk_workspace rule.\n') |
| 729 | with open(_TF_WORKSPACE, 'a') as f: |
| 730 | f.write(""" |
| 731 | android_sdk_repository( |
| 732 | name="androidsdk", |
| 733 | api_level=%s, |
| 734 | path="%s", |
| 735 | build_tools_version="%s")\n |
| 736 | """ % (android_api_level, android_sdk_home_path, android_build_tools_version)) |
| 737 | |
| 738 | |
| 739 | def write_android_ndk_workspace_rule(android_ndk_home_path): |
| 740 | print('Writing android_ndk_workspace rule.') |
| 741 | ndk_api_level = check_ndk_level(android_ndk_home_path) |
| 742 | if int(ndk_api_level) not in _SUPPORTED_ANDROID_NDK_VERSIONS: |
| 743 | print('WARNING: The API level of the NDK in %s is %s, which is not ' |
| 744 | 'supported by Bazel (officially supported versions: %s). Please use ' |
| 745 | 'another version. Compiling Android targets may result in confusing ' |
| 746 | 'errors.\n' % (android_ndk_home_path, ndk_api_level, |
| 747 | _SUPPORTED_ANDROID_NDK_VERSIONS)) |
| 748 | with open(_TF_WORKSPACE, 'a') as f: |
| 749 | f.write(""" |
| 750 | android_ndk_repository( |
| 751 | name="androidndk", |
| 752 | path="%s", |
| 753 | api_level=%s)\n |
| 754 | """ % (android_ndk_home_path, ndk_api_level)) |
| 755 | |
| 756 | |
| 757 | def check_ndk_level(android_ndk_home_path): |
| 758 | """Check the revision number of an Android NDK path.""" |
| 759 | properties_path = '%s/source.properties' % android_ndk_home_path |
| 760 | if is_windows() or is_cygwin(): |
| 761 | properties_path = cygpath(properties_path) |
| 762 | with open(properties_path, 'r') as f: |
| 763 | filedata = f.read() |
| 764 | |
| 765 | revision = re.search(r'Pkg.Revision = (\d+)', filedata) |
| 766 | if revision: |
| 767 | return revision.group(1) |
| 768 | return None |
| 769 | |
| 770 | |
| 771 | def workspace_has_any_android_rule(): |
| 772 | """Check the WORKSPACE for existing android_*_repository rules.""" |
| 773 | with open(_TF_WORKSPACE, 'r') as f: |
| 774 | workspace = f.read() |
| 775 | has_any_rule = re.search(r'^android_[ns]dk_repository', |
| 776 | workspace, |
| 777 | re.MULTILINE) |
| 778 | return has_any_rule |
| 779 | |
| 780 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 781 | def set_gcc_host_compiler_path(environ_cp): |
| 782 | """Set GCC_HOST_COMPILER_PATH.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 783 | default_gcc_host_compiler_path = which('gcc') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 784 | cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH') |
| 785 | |
| 786 | if os.path.islink(cuda_bin_symlink): |
| 787 | # os.readlink is only available in linux |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 788 | default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 789 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 790 | gcc_host_compiler_path = prompt_loop_or_load_from_env( |
| 791 | environ_cp, |
| 792 | var_name='GCC_HOST_COMPILER_PATH', |
| 793 | var_default=default_gcc_host_compiler_path, |
| 794 | ask_for_var= |
| 795 | 'Please specify which gcc should be used by nvcc as the host compiler.', |
| 796 | check_success=os.path.exists, |
| 797 | error_msg='Invalid gcc path. %s cannot be found.', |
| 798 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 799 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 800 | write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path) |
| 801 | |
| 802 | |
| 803 | def set_tf_cuda_version(environ_cp): |
| 804 | """Set CUDA_TOOLKIT_PATH and TF_CUDA_VERSION.""" |
| 805 | ask_cuda_version = ( |
| 806 | 'Please specify the CUDA SDK version you want to use, ' |
| 807 | 'e.g. 7.0. [Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION |
| 808 | |
Austin Anderson | f9a88f8 | 2017-12-13 11:49:40 -0800 | [diff] [blame^] | 809 | for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 810 | # Configure the Cuda SDK version to use. |
| 811 | tf_cuda_version = get_from_env_or_user_or_default( |
| 812 | environ_cp, 'TF_CUDA_VERSION', ask_cuda_version, _DEFAULT_CUDA_VERSION) |
| 813 | |
| 814 | # Find out where the CUDA toolkit is installed |
| 815 | default_cuda_path = _DEFAULT_CUDA_PATH |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 816 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 817 | default_cuda_path = cygpath( |
| 818 | environ_cp.get('CUDA_PATH', _DEFAULT_CUDA_PATH_WIN)) |
| 819 | elif is_linux(): |
| 820 | # If the default doesn't exist, try an alternative default. |
| 821 | if (not os.path.exists(default_cuda_path) |
| 822 | ) and os.path.exists(_DEFAULT_CUDA_PATH_LINUX): |
| 823 | default_cuda_path = _DEFAULT_CUDA_PATH_LINUX |
| 824 | ask_cuda_path = ('Please specify the location where CUDA %s toolkit is' |
| 825 | ' installed. Refer to README.md for more details. ' |
| 826 | '[Default is %s]: ') % (tf_cuda_version, default_cuda_path) |
| 827 | cuda_toolkit_path = get_from_env_or_user_or_default( |
| 828 | environ_cp, 'CUDA_TOOLKIT_PATH', ask_cuda_path, default_cuda_path) |
| 829 | |
| 830 | if is_windows(): |
| 831 | cuda_rt_lib_path = 'lib/x64/cudart.lib' |
| 832 | elif is_linux(): |
| 833 | cuda_rt_lib_path = 'lib64/libcudart.so.%s' % tf_cuda_version |
| 834 | elif is_macos(): |
| 835 | cuda_rt_lib_path = 'lib/libcudart.%s.dylib' % tf_cuda_version |
| 836 | |
| 837 | cuda_toolkit_path_full = os.path.join(cuda_toolkit_path, cuda_rt_lib_path) |
| 838 | if os.path.exists(cuda_toolkit_path_full): |
| 839 | break |
| 840 | |
| 841 | # Reset and retry |
| 842 | print('Invalid path to CUDA %s toolkit. %s cannot be found' % |
| 843 | (tf_cuda_version, cuda_toolkit_path_full)) |
| 844 | environ_cp['TF_CUDA_VERSION'] = '' |
| 845 | environ_cp['CUDA_TOOLKIT_PATH'] = '' |
| 846 | |
Austin Anderson | f9a88f8 | 2017-12-13 11:49:40 -0800 | [diff] [blame^] | 847 | else: |
| 848 | raise UserInputError('Invalid TF_CUDA_SETTING setting was provided %d ' |
| 849 | 'times in a row. Assuming to be a scripting mistake.' % |
| 850 | _DEFAULT_PROMPT_ASK_ATTEMPTS) |
| 851 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 852 | # Set CUDA_TOOLKIT_PATH and TF_CUDA_VERSION |
| 853 | environ_cp['CUDA_TOOLKIT_PATH'] = cuda_toolkit_path |
| 854 | write_action_env_to_bazelrc('CUDA_TOOLKIT_PATH', cuda_toolkit_path) |
| 855 | environ_cp['TF_CUDA_VERSION'] = tf_cuda_version |
| 856 | write_action_env_to_bazelrc('TF_CUDA_VERSION', tf_cuda_version) |
| 857 | |
| 858 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 859 | def set_tf_cudnn_version(environ_cp): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 860 | """Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION.""" |
| 861 | ask_cudnn_version = ( |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 862 | 'Please specify the cuDNN version you want to use. ' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 863 | '[Leave empty to default to cuDNN %s.0]: ') % _DEFAULT_CUDNN_VERSION |
| 864 | |
Austin Anderson | f9a88f8 | 2017-12-13 11:49:40 -0800 | [diff] [blame^] | 865 | for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 866 | tf_cudnn_version = get_from_env_or_user_or_default( |
| 867 | environ_cp, 'TF_CUDNN_VERSION', ask_cudnn_version, |
| 868 | _DEFAULT_CUDNN_VERSION) |
| 869 | |
| 870 | default_cudnn_path = environ_cp.get('CUDA_TOOLKIT_PATH') |
| 871 | ask_cudnn_path = (r'Please specify the location where cuDNN %s library is ' |
| 872 | 'installed. Refer to README.md for more details. [Default' |
| 873 | ' is %s]:') % (tf_cudnn_version, default_cudnn_path) |
| 874 | cudnn_install_path = get_from_env_or_user_or_default( |
| 875 | environ_cp, 'CUDNN_INSTALL_PATH', ask_cudnn_path, default_cudnn_path) |
| 876 | |
| 877 | # Result returned from "read" will be used unexpanded. That make "~" |
| 878 | # unusable. Going through one more level of expansion to handle that. |
| 879 | cudnn_install_path = os.path.realpath( |
| 880 | os.path.expanduser(cudnn_install_path)) |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 881 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 882 | cudnn_install_path = cygpath(cudnn_install_path) |
| 883 | |
| 884 | if is_windows(): |
| 885 | cuda_dnn_lib_path = 'lib/x64/cudnn.lib' |
| 886 | cuda_dnn_lib_alt_path = 'lib/x64/cudnn.lib' |
| 887 | elif is_linux(): |
| 888 | cuda_dnn_lib_path = 'lib64/libcudnn.so.%s' % tf_cudnn_version |
| 889 | cuda_dnn_lib_alt_path = 'libcudnn.so.%s' % tf_cudnn_version |
| 890 | elif is_macos(): |
| 891 | cuda_dnn_lib_path = 'lib/libcudnn.%s.dylib' % tf_cudnn_version |
| 892 | cuda_dnn_lib_alt_path = 'libcudnn.%s.dylib' % tf_cudnn_version |
| 893 | |
| 894 | cuda_dnn_lib_path_full = os.path.join(cudnn_install_path, cuda_dnn_lib_path) |
| 895 | cuda_dnn_lib_alt_path_full = os.path.join(cudnn_install_path, |
| 896 | cuda_dnn_lib_alt_path) |
| 897 | if os.path.exists(cuda_dnn_lib_path_full) or os.path.exists( |
| 898 | cuda_dnn_lib_alt_path_full): |
| 899 | break |
| 900 | |
| 901 | # Try another alternative for Linux |
| 902 | if is_linux(): |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 903 | ldconfig_bin = which('ldconfig') or '/sbin/ldconfig' |
| 904 | cudnn_path_from_ldconfig = run_shell([ldconfig_bin, '-p']) |
| 905 | cudnn_path_from_ldconfig = re.search('.*libcudnn.so .* => (.*)', |
A. Unique TensorFlower | e722358 | 2017-09-06 17:57:04 -0700 | [diff] [blame] | 906 | cudnn_path_from_ldconfig) |
| 907 | if cudnn_path_from_ldconfig: |
| 908 | cudnn_path_from_ldconfig = cudnn_path_from_ldconfig.group(1) |
| 909 | if os.path.exists('%s.%s' % (cudnn_path_from_ldconfig, |
| 910 | tf_cudnn_version)): |
| 911 | cudnn_install_path = os.path.dirname(cudnn_path_from_ldconfig) |
| 912 | break |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 913 | |
| 914 | # Reset and Retry |
| 915 | print( |
| 916 | 'Invalid path to cuDNN %s toolkit. None of the following files can be ' |
| 917 | 'found:' % tf_cudnn_version) |
| 918 | print(cuda_dnn_lib_path_full) |
| 919 | print(cuda_dnn_lib_alt_path_full) |
| 920 | if is_linux(): |
| 921 | print('%s.%s' % (cudnn_path_from_ldconfig, tf_cudnn_version)) |
| 922 | |
| 923 | environ_cp['TF_CUDNN_VERSION'] = '' |
Austin Anderson | f9a88f8 | 2017-12-13 11:49:40 -0800 | [diff] [blame^] | 924 | else: |
| 925 | raise UserInputError('Invalid TF_CUDNN setting was provided %d ' |
| 926 | 'times in a row. Assuming to be a scripting mistake.' % |
| 927 | _DEFAULT_PROMPT_ASK_ATTEMPTS) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 928 | |
| 929 | # Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION |
| 930 | environ_cp['CUDNN_INSTALL_PATH'] = cudnn_install_path |
| 931 | write_action_env_to_bazelrc('CUDNN_INSTALL_PATH', cudnn_install_path) |
| 932 | environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version |
| 933 | write_action_env_to_bazelrc('TF_CUDNN_VERSION', tf_cudnn_version) |
| 934 | |
| 935 | |
| 936 | def get_native_cuda_compute_capabilities(environ_cp): |
| 937 | """Get native cuda compute capabilities. |
| 938 | |
| 939 | Args: |
| 940 | environ_cp: copy of the os.environ. |
| 941 | Returns: |
| 942 | string of native cuda compute capabilities, separated by comma. |
| 943 | """ |
| 944 | device_query_bin = os.path.join( |
| 945 | environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery') |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 946 | if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK): |
| 947 | try: |
| 948 | output = run_shell(device_query_bin).split('\n') |
| 949 | pattern = re.compile('[0-9]*\\.[0-9]*') |
| 950 | output = [pattern.search(x) for x in output if 'Capability' in x] |
| 951 | output = ','.join(x.group() for x in output if x is not None) |
| 952 | except subprocess.CalledProcessError: |
| 953 | output = '' |
| 954 | else: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 955 | output = '' |
| 956 | return output |
| 957 | |
| 958 | |
| 959 | def set_tf_cuda_compute_capabilities(environ_cp): |
| 960 | """Set TF_CUDA_COMPUTE_CAPABILITIES.""" |
| 961 | while True: |
| 962 | native_cuda_compute_capabilities = get_native_cuda_compute_capabilities( |
| 963 | environ_cp) |
| 964 | if not native_cuda_compute_capabilities: |
| 965 | default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES |
| 966 | else: |
| 967 | default_cuda_compute_capabilities = native_cuda_compute_capabilities |
| 968 | |
| 969 | ask_cuda_compute_capabilities = ( |
| 970 | 'Please specify a list of comma-separated ' |
| 971 | 'Cuda compute capabilities you want to ' |
| 972 | 'build with.\nYou can find the compute ' |
| 973 | 'capability of your device at: ' |
| 974 | 'https://developer.nvidia.com/cuda-gpus.\nPlease' |
| 975 | ' note that each additional compute ' |
| 976 | 'capability significantly increases your ' |
| 977 | 'build time and binary size. [Default is: %s]' % |
| 978 | default_cuda_compute_capabilities) |
| 979 | tf_cuda_compute_capabilities = get_from_env_or_user_or_default( |
| 980 | environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES', |
| 981 | ask_cuda_compute_capabilities, default_cuda_compute_capabilities) |
| 982 | # Check whether all capabilities from the input is valid |
| 983 | all_valid = True |
| 984 | for compute_capability in tf_cuda_compute_capabilities.split(','): |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 985 | m = re.match('[0-9]+.[0-9]+', compute_capability) |
| 986 | if not m: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 987 | print('Invalid compute capability: ' % compute_capability) |
| 988 | all_valid = False |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 989 | else: |
| 990 | ver = int(m.group(0).split('.')[0]) |
| 991 | if ver < 3: |
| 992 | print('Only compute capabilities 3.0 or higher are supported.') |
| 993 | all_valid = False |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 994 | |
| 995 | if all_valid: |
| 996 | break |
| 997 | |
| 998 | # Reset and Retry |
| 999 | environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = '' |
| 1000 | |
| 1001 | # Set TF_CUDA_COMPUTE_CAPABILITIES |
| 1002 | environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities |
| 1003 | write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES', |
| 1004 | tf_cuda_compute_capabilities) |
| 1005 | |
| 1006 | |
| 1007 | def set_other_cuda_vars(environ_cp): |
| 1008 | """Set other CUDA related variables.""" |
| 1009 | if is_windows(): |
| 1010 | # The following three variables are needed for MSVC toolchain configuration |
| 1011 | # in Bazel |
| 1012 | environ_cp['CUDA_PATH'] = environ_cp.get('CUDA_TOOLKIT_PATH') |
| 1013 | environ_cp['CUDA_COMPUTE_CAPABILITIES'] = environ_cp.get( |
| 1014 | 'TF_CUDA_COMPUTE_CAPABILITIES') |
| 1015 | environ_cp['NO_WHOLE_ARCHIVE_OPTION'] = 1 |
| 1016 | write_action_env_to_bazelrc('CUDA_PATH', environ_cp.get('CUDA_PATH')) |
| 1017 | write_action_env_to_bazelrc('CUDA_COMPUTE_CAPABILITIE', |
| 1018 | environ_cp.get('CUDA_COMPUTE_CAPABILITIE')) |
| 1019 | write_action_env_to_bazelrc('NO_WHOLE_ARCHIVE_OPTION', |
| 1020 | environ_cp.get('NO_WHOLE_ARCHIVE_OPTION')) |
| 1021 | write_to_bazelrc('build --config=win-cuda') |
| 1022 | write_to_bazelrc('test --config=win-cuda') |
| 1023 | else: |
| 1024 | # If CUDA is enabled, always use GPU during build and test. |
| 1025 | if environ_cp.get('TF_CUDA_CLANG') == '1': |
| 1026 | write_to_bazelrc('build --config=cuda_clang') |
| 1027 | write_to_bazelrc('test --config=cuda_clang') |
| 1028 | else: |
| 1029 | write_to_bazelrc('build --config=cuda') |
| 1030 | write_to_bazelrc('test --config=cuda') |
| 1031 | |
| 1032 | |
| 1033 | def set_host_cxx_compiler(environ_cp): |
| 1034 | """Set HOST_CXX_COMPILER.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1035 | default_cxx_host_compiler = which('g++') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1036 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1037 | host_cxx_compiler = prompt_loop_or_load_from_env( |
| 1038 | environ_cp, |
| 1039 | var_name='HOST_CXX_COMPILER', |
| 1040 | var_default=default_cxx_host_compiler, |
| 1041 | ask_for_var=('Please specify which C++ compiler should be used as the ' |
| 1042 | 'host C++ compiler.'), |
| 1043 | check_success=os.path.exists, |
| 1044 | error_msg='Invalid C++ compiler path. %s cannot be found.', |
| 1045 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1046 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1047 | write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler) |
| 1048 | |
| 1049 | |
| 1050 | def set_host_c_compiler(environ_cp): |
| 1051 | """Set HOST_C_COMPILER.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1052 | default_c_host_compiler = which('gcc') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1053 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1054 | host_c_compiler = prompt_loop_or_load_from_env( |
| 1055 | environ_cp, |
| 1056 | var_name='HOST_C_COMPILER', |
| 1057 | var_default=default_c_host_compiler, |
| 1058 | ask_for_var=('Please specify which C compiler should be used as the host' |
| 1059 | 'C compiler.'), |
| 1060 | check_success=os.path.exists, |
| 1061 | error_msg='Invalid C compiler path. %s cannot be found.', |
| 1062 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1063 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1064 | write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler) |
| 1065 | |
| 1066 | |
| 1067 | def set_computecpp_toolkit_path(environ_cp): |
| 1068 | """Set COMPUTECPP_TOOLKIT_PATH.""" |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1069 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1070 | def toolkit_exists(toolkit_path): |
| 1071 | """Check if a computecpp toolkit path is valid.""" |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1072 | if is_linux(): |
| 1073 | sycl_rt_lib_path = 'lib/libComputeCpp.so' |
| 1074 | else: |
| 1075 | sycl_rt_lib_path = '' |
| 1076 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1077 | sycl_rt_lib_path_full = os.path.join(toolkit_path, |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1078 | sycl_rt_lib_path) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1079 | exists = os.path.exists(sycl_rt_lib_path_full) |
| 1080 | if not exists: |
| 1081 | print('Invalid SYCL %s library path. %s cannot be found' % |
| 1082 | (_TF_OPENCL_VERSION, sycl_rt_lib_path_full)) |
| 1083 | return exists |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1084 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1085 | computecpp_toolkit_path = prompt_loop_or_load_from_env( |
| 1086 | environ_cp, |
| 1087 | var_name='COMPUTECPP_TOOLKIT_PATH', |
| 1088 | var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH, |
| 1089 | ask_for_var=( |
| 1090 | 'Please specify the location where ComputeCpp for SYCL %s is ' |
| 1091 | 'installed.' % _TF_OPENCL_VERSION), |
| 1092 | check_success=toolkit_exists, |
| 1093 | error_msg='Invalid SYCL compiler path. %s cannot be found.', |
| 1094 | suppress_default_error=True) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1095 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1096 | write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH', |
| 1097 | computecpp_toolkit_path) |
| 1098 | |
| 1099 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1100 | def set_trisycl_include_dir(environ_cp): |
| 1101 | """Set TRISYCL_INCLUDE_DIR.""" |
Shanqing Cai | fe84061 | 2017-12-06 18:43:24 -0800 | [diff] [blame] | 1102 | ask_trisycl_include_dir = ('Please specify the location of the triSYCL ' |
| 1103 | 'include directory. (Use --config=sycl_trisycl ' |
| 1104 | 'when building with Bazel) ' |
| 1105 | '[Default is %s]: ') % _DEFAULT_TRISYCL_INCLUDE_DIR |
| 1106 | while True: |
| 1107 | trisycl_include_dir = get_from_env_or_user_or_default( |
| 1108 | environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir, |
| 1109 | _DEFAULT_TRISYCL_INCLUDE_DIR) |
| 1110 | if os.path.exists(trisycl_include_dir): |
| 1111 | break |
| 1112 | |
| 1113 | print('Invalid triSYCL include directory, %s cannot be found' |
| 1114 | % (trisycl_include_dir)) |
| 1115 | |
| 1116 | # Set TRISYCL_INCLUDE_DIR |
| 1117 | environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir |
| 1118 | write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', |
| 1119 | trisycl_include_dir) |
| 1120 | |
| 1121 | |
| 1122 | def set_trisycl_include_dir(environ_cp): |
| 1123 | """Set TRISYCL_INCLUDE_DIR.""" |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1124 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1125 | trisycl_include_dir = prompt_loop_or_load_from_env( |
| 1126 | environ_cp, |
| 1127 | var_name='TRISYCL_INCLUDE_DIR', |
| 1128 | var_default=_DEFAULT_TRISYCL_INCLUDE_DIR, |
| 1129 | ask_for_var=('Please specify the location of the triSYCL include ' |
| 1130 | 'directory. (Use --config=sycl_trisycl when building with ' |
| 1131 | 'Bazel)'), |
| 1132 | check_success=os.path.exists, |
| 1133 | error_msg='Invalid trySYCL include directory. %s cannot be found.', |
| 1134 | suppress_default_error=True) |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1135 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1136 | write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', trisycl_include_dir) |
| 1137 | |
| 1138 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1139 | def set_mpi_home(environ_cp): |
| 1140 | """Set MPI_HOME.""" |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1141 | |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1142 | default_mpi_home = which('mpirun') or which('mpiexec') or '' |
| 1143 | default_mpi_home = os.path.dirname(os.path.dirname(default_mpi_home)) |
| 1144 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1145 | def valid_mpi_path(mpi_home): |
| 1146 | exists = (os.path.exists(os.path.join(mpi_home, 'include')) and |
| 1147 | os.path.exists(os.path.join(mpi_home, 'lib'))) |
| 1148 | if not exists: |
| 1149 | print('Invalid path to the MPI Toolkit. %s or %s cannot be found' % |
| 1150 | (os.path.join(mpi_home, 'include'), |
| 1151 | os.path.exists(os.path.join(mpi_home, 'lib')))) |
| 1152 | return exists |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1153 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1154 | _ = prompt_loop_or_load_from_env( |
| 1155 | environ_cp, |
| 1156 | var_name='MPI_HOME', |
| 1157 | var_default=default_mpi_home, |
| 1158 | ask_for_var='Please specify the MPI toolkit folder.', |
| 1159 | check_success=valid_mpi_path, |
| 1160 | error_msg='', |
| 1161 | suppress_default_error=True) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1162 | |
| 1163 | |
| 1164 | def set_other_mpi_vars(environ_cp): |
| 1165 | """Set other MPI related variables.""" |
| 1166 | # Link the MPI header files |
| 1167 | mpi_home = environ_cp.get('MPI_HOME') |
| 1168 | symlink_force('%s/include/mpi.h' % mpi_home, 'third_party/mpi/mpi.h') |
| 1169 | |
| 1170 | # Determine if we use OpenMPI or MVAPICH, these require different header files |
| 1171 | # to be included here to make bazel dependency checker happy |
| 1172 | if os.path.exists(os.path.join(mpi_home, 'include/mpi_portable_platform.h')): |
| 1173 | symlink_force( |
| 1174 | os.path.join(mpi_home, 'include/mpi_portable_platform.h'), |
| 1175 | 'third_party/mpi/mpi_portable_platform.h') |
| 1176 | # TODO(gunan): avoid editing files in configure |
| 1177 | sed_in_place('third_party/mpi/mpi.bzl', 'MPI_LIB_IS_OPENMPI=False', |
| 1178 | 'MPI_LIB_IS_OPENMPI=True') |
| 1179 | else: |
| 1180 | # MVAPICH / MPICH |
| 1181 | symlink_force( |
| 1182 | os.path.join(mpi_home, 'include/mpio.h'), 'third_party/mpi/mpio.h') |
| 1183 | symlink_force( |
| 1184 | os.path.join(mpi_home, 'include/mpicxx.h'), 'third_party/mpi/mpicxx.h') |
| 1185 | # TODO(gunan): avoid editing files in configure |
| 1186 | sed_in_place('third_party/mpi/mpi.bzl', 'MPI_LIB_IS_OPENMPI=True', |
| 1187 | 'MPI_LIB_IS_OPENMPI=False') |
| 1188 | |
| 1189 | if os.path.exists(os.path.join(mpi_home, 'lib/libmpi.so')): |
| 1190 | symlink_force( |
| 1191 | os.path.join(mpi_home, 'lib/libmpi.so'), 'third_party/mpi/libmpi.so') |
| 1192 | else: |
| 1193 | raise ValueError('Cannot find the MPI library file in %s/lib' % mpi_home) |
| 1194 | |
| 1195 | |
| 1196 | def set_mkl(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1197 | write_to_bazelrc('build:mkl --define using_mkl=true') |
| 1198 | write_to_bazelrc('build:mkl -c opt') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1199 | print( |
| 1200 | 'Add "--config=mkl" to your bazel command to build with MKL ' |
| 1201 | 'support.\nPlease note that MKL on MacOS or windows is still not ' |
| 1202 | 'supported.\nIf you would like to use a local MKL instead of ' |
| 1203 | 'downloading, please set the environment variable \"TF_MKL_ROOT\" every ' |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1204 | 'time before build.\n') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1205 | |
| 1206 | |
Allen Lavoie | 5c7f9e3 | 2017-09-21 11:29:45 -0700 | [diff] [blame] | 1207 | def set_monolithic(): |
| 1208 | # Add --config=monolithic to your bazel command to use a mostly-static |
| 1209 | # build and disable modular op registration support (this will revert to |
| 1210 | # loading TensorFlow with RTLD_GLOBAL in Python). By default (without |
| 1211 | # --config=monolithic), TensorFlow will build with a dependence on |
| 1212 | # //tensorflow:libtensorflow_framework.so. |
| 1213 | write_to_bazelrc('build:monolithic --define framework_shared_object=false') |
| 1214 | # For projects which use TensorFlow as part of a Bazel build process, putting |
| 1215 | # nothing in a bazelrc will default to a monolithic build. The following line |
| 1216 | # opts in to modular op registration support by default: |
| 1217 | write_to_bazelrc('build --define framework_shared_object=true') |
| 1218 | |
| 1219 | |
Michael Case | f1ecdd6 | 2017-10-24 18:07:59 -0700 | [diff] [blame] | 1220 | def create_android_bazelrc_configs(): |
| 1221 | # Flags for --config=android |
| 1222 | write_to_bazelrc('build:android --crosstool_top=//external:android/crosstool') |
| 1223 | write_to_bazelrc( |
| 1224 | 'build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain') |
| 1225 | # Flags for --config=android_arm |
| 1226 | write_to_bazelrc('build:android_arm --config=android') |
| 1227 | write_to_bazelrc('build:android_arm --cpu=armeabi-v7a') |
| 1228 | # Flags for --config=android_arm64 |
| 1229 | write_to_bazelrc('build:android_arm64 --config=android') |
| 1230 | write_to_bazelrc('build:android_arm64 --cpu=arm64-v8a') |
| 1231 | |
| 1232 | |
A. Unique TensorFlower | 061c359 | 2017-11-13 14:21:04 -0800 | [diff] [blame] | 1233 | def set_grpc_build_flags(): |
| 1234 | write_to_bazelrc('build --define grpc_no_ares=true') |
| 1235 | |
| 1236 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1237 | def main(): |
| 1238 | # Make a copy of os.environ to be clear when functions and getting and setting |
| 1239 | # environment variables. |
| 1240 | environ_cp = dict(os.environ) |
| 1241 | |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 1242 | check_bazel_version('0.5.4') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1243 | |
| 1244 | reset_tf_configure_bazelrc() |
| 1245 | cleanup_makefile() |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 1246 | setup_python(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1247 | run_gen_git_source(environ_cp) |
| 1248 | |
| 1249 | if is_windows(): |
Benoit Steiner | 355e25e | 2017-10-24 19:47:46 -0700 | [diff] [blame] | 1250 | environ_cp['TF_NEED_S3'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1251 | environ_cp['TF_NEED_GCP'] = '0' |
| 1252 | environ_cp['TF_NEED_HDFS'] = '0' |
| 1253 | environ_cp['TF_NEED_JEMALLOC'] = '0' |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1254 | environ_cp['TF_NEED_OPENCL_SYCL'] = '0' |
| 1255 | environ_cp['TF_NEED_COMPUTECPP'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1256 | environ_cp['TF_NEED_OPENCL'] = '0' |
| 1257 | environ_cp['TF_CUDA_CLANG'] = '0' |
| 1258 | |
| 1259 | if is_macos(): |
| 1260 | environ_cp['TF_NEED_JEMALLOC'] = '0' |
| 1261 | |
| 1262 | set_build_var(environ_cp, 'TF_NEED_JEMALLOC', 'jemalloc as malloc', |
| 1263 | 'with_jemalloc', True) |
| 1264 | set_build_var(environ_cp, 'TF_NEED_GCP', 'Google Cloud Platform', |
Benoit Steiner | 355e25e | 2017-10-24 19:47:46 -0700 | [diff] [blame] | 1265 | 'with_gcp_support', True, 'gcp') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1266 | set_build_var(environ_cp, 'TF_NEED_HDFS', 'Hadoop File System', |
Benoit Steiner | 355e25e | 2017-10-24 19:47:46 -0700 | [diff] [blame] | 1267 | 'with_hdfs_support', True, 'hdfs') |
Michael Case | 7e4e336 | 2017-10-09 13:31:15 -0700 | [diff] [blame] | 1268 | set_build_var(environ_cp, 'TF_NEED_S3', 'Amazon S3 File System', |
| 1269 | 'with_s3_support', True, 's3') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1270 | set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support', |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 1271 | False, 'xla') |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 1272 | set_build_var(environ_cp, 'TF_NEED_GDR', 'GDR', 'with_gdr_support', |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 1273 | False, 'gdr') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1274 | set_build_var(environ_cp, 'TF_NEED_VERBS', 'VERBS', 'with_verbs_support', |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 1275 | False, 'verbs') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1276 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1277 | set_action_env_var(environ_cp, 'TF_NEED_OPENCL_SYCL', 'OpenCL SYCL', False) |
| 1278 | if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1279 | set_host_cxx_compiler(environ_cp) |
| 1280 | set_host_c_compiler(environ_cp) |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1281 | set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True) |
| 1282 | if environ_cp.get('TF_NEED_COMPUTECPP') == '1': |
| 1283 | set_computecpp_toolkit_path(environ_cp) |
| 1284 | else: |
| 1285 | set_trisycl_include_dir(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1286 | |
| 1287 | set_action_env_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False) |
A. Unique TensorFlower | 24cbb2a | 2017-09-08 07:45:44 -0700 | [diff] [blame] | 1288 | if (environ_cp.get('TF_NEED_CUDA') == '1' and |
| 1289 | 'TF_CUDA_CONFIG_REPO' not in environ_cp): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1290 | set_tf_cuda_version(environ_cp) |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1291 | set_tf_cudnn_version(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1292 | set_tf_cuda_compute_capabilities(environ_cp) |
| 1293 | |
| 1294 | set_tf_cuda_clang(environ_cp) |
| 1295 | if environ_cp.get('TF_CUDA_CLANG') == '1': |
| 1296 | # Set up which clang we should use as the cuda / host compiler. |
| 1297 | set_clang_cuda_compiler_path(environ_cp) |
| 1298 | else: |
| 1299 | # Set up which gcc nvcc should use as the host compiler |
| 1300 | # No need to set this on Windows |
| 1301 | if not is_windows(): |
| 1302 | set_gcc_host_compiler_path(environ_cp) |
| 1303 | set_other_cuda_vars(environ_cp) |
| 1304 | |
| 1305 | set_build_var(environ_cp, 'TF_NEED_MPI', 'MPI', 'with_mpi_support', False) |
| 1306 | if environ_cp.get('TF_NEED_MPI') == '1': |
| 1307 | set_mpi_home(environ_cp) |
| 1308 | set_other_mpi_vars(environ_cp) |
| 1309 | |
A. Unique TensorFlower | 061c359 | 2017-11-13 14:21:04 -0800 | [diff] [blame] | 1310 | set_grpc_build_flags() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1311 | set_cc_opt_flags(environ_cp) |
| 1312 | set_mkl() |
Allen Lavoie | 5c7f9e3 | 2017-09-21 11:29:45 -0700 | [diff] [blame] | 1313 | set_monolithic() |
Michael Case | f1ecdd6 | 2017-10-24 18:07:59 -0700 | [diff] [blame] | 1314 | create_android_bazelrc_configs() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1315 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1316 | if workspace_has_any_android_rule(): |
| 1317 | print('The WORKSPACE file has at least one of ["android_sdk_repository", ' |
| 1318 | '"android_ndk_repository"] already set. Will not ask to help ' |
| 1319 | 'configure the WORKSPACE. Please delete the existing rules to ' |
| 1320 | 'activate the helper.\n') |
| 1321 | else: |
| 1322 | if get_var( |
| 1323 | environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', |
| 1324 | False, |
| 1325 | ('Would you like to interactively configure ./WORKSPACE for ' |
| 1326 | 'Android builds?'), |
| 1327 | 'Searching for NDK and SDK installations.', |
| 1328 | 'Not configuring the WORKSPACE for Android builds.'): |
| 1329 | create_android_ndk_rule(environ_cp) |
| 1330 | create_android_sdk_rule(environ_cp) |
| 1331 | |
| 1332 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1333 | if __name__ == '__main__': |
| 1334 | main() |