A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | # ============================================================================== |
| 15 | """configure script to get build parameters from user.""" |
| 16 | |
| 17 | from __future__ import absolute_import |
| 18 | from __future__ import division |
| 19 | from __future__ import print_function |
| 20 | |
Michael Case | 671baf0 | 2018-02-21 20:39:21 -0800 | [diff] [blame] | 21 | import argparse |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 22 | import errno |
| 23 | import os |
| 24 | import platform |
| 25 | import re |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 26 | import subprocess |
| 27 | import sys |
| 28 | |
Andrew Selle | c9885ea | 2017-11-06 09:37:03 -0800 | [diff] [blame] | 29 | # pylint: disable=g-import-not-at-top |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 30 | try: |
| 31 | from shutil import which |
| 32 | except ImportError: |
| 33 | from distutils.spawn import find_executable as which |
Andrew Selle | c9885ea | 2017-11-06 09:37:03 -0800 | [diff] [blame] | 34 | # pylint: enable=g-import-not-at-top |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 35 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 36 | _DEFAULT_CUDA_VERSION = '9.0' |
| 37 | _DEFAULT_CUDNN_VERSION = '7' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 38 | _DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,5.2' |
| 39 | _DEFAULT_CUDA_PATH = '/usr/local/cuda' |
| 40 | _DEFAULT_CUDA_PATH_LINUX = '/opt/cuda' |
| 41 | _DEFAULT_CUDA_PATH_WIN = ('C:/Program Files/NVIDIA GPU Computing ' |
| 42 | 'Toolkit/CUDA/v%s' % _DEFAULT_CUDA_VERSION) |
Sami Kama | 825e7a3 | 2018-01-19 22:58:50 +0000 | [diff] [blame] | 43 | _DEFAULT_TENSORRT_PATH_LINUX = '/usr/lib/x86_64-linux-gnu' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 44 | _TF_OPENCL_VERSION = '1.2' |
| 45 | _DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp' |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 46 | _DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include' |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 47 | _SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15] |
| 48 | |
| 49 | _DEFAULT_PROMPT_ASK_ATTEMPTS = 10 |
| 50 | |
Michael Case | 671baf0 | 2018-02-21 20:39:21 -0800 | [diff] [blame] | 51 | _TF_WORKSPACE_ROOT = os.path.abspath(os.path.dirname(__file__)) |
| 52 | _TF_BAZELRC_FILENAME = '.tf_configure.bazelrc' |
| 53 | _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME) |
| 54 | _TF_WORKSPACE = os.path.join(_TF_WORKSPACE_ROOT, 'WORKSPACE') |
| 55 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 56 | |
| 57 | class UserInputError(Exception): |
| 58 | pass |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 59 | |
| 60 | |
| 61 | def is_windows(): |
| 62 | return platform.system() == 'Windows' |
| 63 | |
| 64 | |
| 65 | def is_linux(): |
| 66 | return platform.system() == 'Linux' |
| 67 | |
| 68 | |
| 69 | def is_macos(): |
| 70 | return platform.system() == 'Darwin' |
| 71 | |
| 72 | |
| 73 | def is_ppc64le(): |
| 74 | return platform.machine() == 'ppc64le' |
| 75 | |
| 76 | |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 77 | def is_cygwin(): |
| 78 | return platform.system().startswith('CYGWIN_NT') |
| 79 | |
| 80 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 81 | def get_input(question): |
| 82 | try: |
| 83 | try: |
| 84 | answer = raw_input(question) |
| 85 | except NameError: |
| 86 | answer = input(question) # pylint: disable=bad-builtin |
| 87 | except EOFError: |
| 88 | answer = '' |
| 89 | return answer |
| 90 | |
| 91 | |
| 92 | def symlink_force(target, link_name): |
| 93 | """Force symlink, equivalent of 'ln -sf'. |
| 94 | |
| 95 | Args: |
| 96 | target: items to link to. |
| 97 | link_name: name of the link. |
| 98 | """ |
| 99 | try: |
| 100 | os.symlink(target, link_name) |
| 101 | except OSError as e: |
| 102 | if e.errno == errno.EEXIST: |
| 103 | os.remove(link_name) |
| 104 | os.symlink(target, link_name) |
| 105 | else: |
| 106 | raise e |
| 107 | |
| 108 | |
| 109 | def sed_in_place(filename, old, new): |
| 110 | """Replace old string with new string in file. |
| 111 | |
| 112 | Args: |
| 113 | filename: string for filename. |
| 114 | old: string to replace. |
| 115 | new: new string to replace to. |
| 116 | """ |
| 117 | with open(filename, 'r') as f: |
| 118 | filedata = f.read() |
| 119 | newdata = filedata.replace(old, new) |
| 120 | with open(filename, 'w') as f: |
| 121 | f.write(newdata) |
| 122 | |
| 123 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 124 | def write_to_bazelrc(line): |
| 125 | with open(_TF_BAZELRC, 'a') as f: |
| 126 | f.write(line + '\n') |
| 127 | |
| 128 | |
| 129 | def write_action_env_to_bazelrc(var_name, var): |
| 130 | write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var))) |
| 131 | |
| 132 | |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 133 | def run_shell(cmd, allow_non_zero=False): |
| 134 | if allow_non_zero: |
| 135 | try: |
| 136 | output = subprocess.check_output(cmd) |
| 137 | except subprocess.CalledProcessError as e: |
| 138 | output = e.output |
| 139 | else: |
| 140 | output = subprocess.check_output(cmd) |
| 141 | return output.decode('UTF-8').strip() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 142 | |
| 143 | |
| 144 | def cygpath(path): |
| 145 | """Convert path from posix to windows.""" |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 146 | return os.path.abspath(path).replace('\\', '/') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 147 | |
| 148 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 149 | def get_python_path(environ_cp, python_bin_path): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 150 | """Get the python site package paths.""" |
| 151 | python_paths = [] |
| 152 | if environ_cp.get('PYTHONPATH'): |
| 153 | python_paths = environ_cp.get('PYTHONPATH').split(':') |
| 154 | try: |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 155 | library_paths = run_shell( |
| 156 | [python_bin_path, '-c', |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 157 | 'import site; print("\\n".join(site.getsitepackages()))']).split('\n') |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 158 | except subprocess.CalledProcessError: |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 159 | library_paths = [run_shell( |
| 160 | [python_bin_path, '-c', |
| 161 | 'from distutils.sysconfig import get_python_lib;' |
| 162 | 'print(get_python_lib())'])] |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 163 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 164 | all_paths = set(python_paths + library_paths) |
| 165 | |
| 166 | paths = [] |
| 167 | for path in all_paths: |
| 168 | if os.path.isdir(path): |
| 169 | paths.append(path) |
| 170 | return paths |
| 171 | |
| 172 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 173 | def get_python_major_version(python_bin_path): |
| 174 | """Get the python major version.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 175 | return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])']) |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 176 | |
| 177 | |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 178 | def setup_python(environ_cp): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 179 | """Setup python related env variables.""" |
| 180 | # Get PYTHON_BIN_PATH, default is the current running python. |
| 181 | default_python_bin_path = sys.executable |
| 182 | ask_python_bin_path = ('Please specify the location of python. [Default is ' |
| 183 | '%s]: ') % default_python_bin_path |
| 184 | while True: |
| 185 | python_bin_path = get_from_env_or_user_or_default( |
| 186 | environ_cp, 'PYTHON_BIN_PATH', ask_python_bin_path, |
| 187 | default_python_bin_path) |
| 188 | # Check if the path is valid |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 189 | if os.path.isfile(python_bin_path) and os.access( |
| 190 | python_bin_path, os.X_OK): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 191 | break |
| 192 | elif not os.path.exists(python_bin_path): |
| 193 | print('Invalid python path: %s cannot be found.' % python_bin_path) |
| 194 | else: |
| 195 | print('%s is not executable. Is it the python binary?' % python_bin_path) |
| 196 | environ_cp['PYTHON_BIN_PATH'] = '' |
| 197 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 198 | # Convert python path to Windows style before checking lib and version |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 199 | if is_windows() or is_cygwin(): |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 200 | python_bin_path = cygpath(python_bin_path) |
| 201 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 202 | # Get PYTHON_LIB_PATH |
| 203 | python_lib_path = environ_cp.get('PYTHON_LIB_PATH') |
| 204 | if not python_lib_path: |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 205 | python_lib_paths = get_python_path(environ_cp, python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 206 | if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1': |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 207 | python_lib_path = python_lib_paths[0] |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 208 | else: |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 209 | print('Found possible Python library paths:\n %s' % |
| 210 | '\n '.join(python_lib_paths)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 211 | default_python_lib_path = python_lib_paths[0] |
| 212 | python_lib_path = get_input( |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 213 | 'Please input the desired Python library path to use. ' |
| 214 | 'Default is [%s]\n' % python_lib_paths[0]) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 215 | if not python_lib_path: |
| 216 | python_lib_path = default_python_lib_path |
Vijay Vasudevan | a1fba7f | 2017-07-28 10:58:56 -0700 | [diff] [blame] | 217 | environ_cp['PYTHON_LIB_PATH'] = python_lib_path |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 218 | |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 219 | python_major_version = get_python_major_version(python_bin_path) |
| 220 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 221 | # Convert python path to Windows style before writing into bazel.rc |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 222 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 223 | python_lib_path = cygpath(python_lib_path) |
| 224 | |
| 225 | # Set-up env variables used by python_configure.bzl |
| 226 | write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path) |
| 227 | write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 228 | write_to_bazelrc('build --force_python=py%s' % python_major_version) |
| 229 | write_to_bazelrc('build --host_force_python=py%s' % python_major_version) |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 230 | write_to_bazelrc('build --python_path=\"%s"' % python_bin_path) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 231 | environ_cp['PYTHON_BIN_PATH'] = python_bin_path |
| 232 | |
| 233 | # Write tools/python_bin_path.sh |
Michael Case | 671baf0 | 2018-02-21 20:39:21 -0800 | [diff] [blame] | 234 | with open(os.path.join( |
| 235 | _TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'), 'w') as f: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 236 | f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path) |
| 237 | |
| 238 | |
Michael Case | 671baf0 | 2018-02-21 20:39:21 -0800 | [diff] [blame] | 239 | def reset_tf_configure_bazelrc(workspace_path): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 240 | """Reset file that contains customized config settings.""" |
| 241 | open(_TF_BAZELRC, 'w').close() |
Michael Case | 671baf0 | 2018-02-21 20:39:21 -0800 | [diff] [blame] | 242 | bazelrc_path = os.path.join(workspace_path, '.bazelrc') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 243 | |
Michael Case | 671baf0 | 2018-02-21 20:39:21 -0800 | [diff] [blame] | 244 | data = [] |
| 245 | if os.path.exists(bazelrc_path): |
| 246 | with open(bazelrc_path, 'r') as f: |
| 247 | data = f.read().splitlines() |
| 248 | with open(bazelrc_path, 'w') as f: |
| 249 | for l in data: |
| 250 | if _TF_BAZELRC_FILENAME in l: |
| 251 | continue |
| 252 | f.write('%s\n' % l) |
| 253 | f.write('import %s\n' % _TF_BAZELRC) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 254 | |
| 255 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 256 | def cleanup_makefile(): |
| 257 | """Delete any leftover BUILD files from the Makefile build. |
| 258 | |
| 259 | These files could interfere with Bazel parsing. |
| 260 | """ |
Michael Case | 671baf0 | 2018-02-21 20:39:21 -0800 | [diff] [blame] | 261 | makefile_download_dir = os.path.join( |
| 262 | _TF_WORKSPACE_ROOT, 'tensorflow', 'contrib', 'makefile', 'downloads') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 263 | if os.path.isdir(makefile_download_dir): |
| 264 | for root, _, filenames in os.walk(makefile_download_dir): |
| 265 | for f in filenames: |
| 266 | if f.endswith('BUILD'): |
| 267 | os.remove(os.path.join(root, f)) |
| 268 | |
| 269 | |
| 270 | def get_var(environ_cp, |
| 271 | var_name, |
| 272 | query_item, |
| 273 | enabled_by_default, |
| 274 | question=None, |
| 275 | yes_reply=None, |
| 276 | no_reply=None): |
| 277 | """Get boolean input from user. |
| 278 | |
| 279 | If var_name is not set in env, ask user to enable query_item or not. If the |
| 280 | response is empty, use the default. |
| 281 | |
| 282 | Args: |
| 283 | environ_cp: copy of the os.environ. |
| 284 | var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". |
| 285 | query_item: string for feature related to the variable, e.g. "Hadoop File |
| 286 | System". |
| 287 | enabled_by_default: boolean for default behavior. |
| 288 | question: optional string for how to ask for user input. |
elilienstein | 6b23b78 | 2018-01-29 15:27:57 -0800 | [diff] [blame] | 289 | yes_reply: optional string for reply when feature is enabled. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 290 | no_reply: optional string for reply when feature is disabled. |
| 291 | |
| 292 | Returns: |
| 293 | boolean value of the variable. |
Philip Yang | 136697e | 2018-01-02 17:13:01 -0800 | [diff] [blame] | 294 | |
| 295 | Raises: |
| 296 | UserInputError: if an environment variable is set, but it cannot be |
| 297 | interpreted as a boolean indicator, assume that the user has made a |
| 298 | scripting error, and will continue to provide invalid input. |
| 299 | Raise the error to avoid infinitely looping. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 300 | """ |
| 301 | if not question: |
| 302 | question = 'Do you wish to build TensorFlow with %s support?' % query_item |
| 303 | if not yes_reply: |
| 304 | yes_reply = '%s support will be enabled for TensorFlow.' % query_item |
| 305 | if not no_reply: |
| 306 | no_reply = 'No %s' % yes_reply |
| 307 | |
| 308 | yes_reply += '\n' |
| 309 | no_reply += '\n' |
| 310 | |
| 311 | if enabled_by_default: |
| 312 | question += ' [Y/n]: ' |
| 313 | else: |
| 314 | question += ' [y/N]: ' |
| 315 | |
| 316 | var = environ_cp.get(var_name) |
Philip Yang | 136697e | 2018-01-02 17:13:01 -0800 | [diff] [blame] | 317 | if var is not None: |
| 318 | var_content = var.strip().lower() |
Shanqing Cai | 805c17c | 2018-01-04 11:04:32 -0500 | [diff] [blame] | 319 | true_strings = ('1', 't', 'true', 'y', 'yes') |
| 320 | false_strings = ('0', 'f', 'false', 'n', 'no') |
| 321 | if var_content in true_strings: |
Philip Yang | 136697e | 2018-01-02 17:13:01 -0800 | [diff] [blame] | 322 | var = True |
Shanqing Cai | 805c17c | 2018-01-04 11:04:32 -0500 | [diff] [blame] | 323 | elif var_content in false_strings: |
Philip Yang | 136697e | 2018-01-02 17:13:01 -0800 | [diff] [blame] | 324 | var = False |
| 325 | else: |
Shanqing Cai | 805c17c | 2018-01-04 11:04:32 -0500 | [diff] [blame] | 326 | raise UserInputError( |
| 327 | 'Environment variable %s must be set as a boolean indicator.\n' |
| 328 | 'The following are accepted as TRUE : %s.\n' |
| 329 | 'The following are accepted as FALSE: %s.\n' |
| 330 | 'Current value is %s.' % ( |
| 331 | var_name, ', '.join(true_strings), ', '.join(false_strings), |
| 332 | var)) |
Philip Yang | 136697e | 2018-01-02 17:13:01 -0800 | [diff] [blame] | 333 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 334 | while var is None: |
| 335 | user_input_origin = get_input(question) |
| 336 | user_input = user_input_origin.strip().lower() |
| 337 | if user_input == 'y': |
| 338 | print(yes_reply) |
| 339 | var = True |
| 340 | elif user_input == 'n': |
| 341 | print(no_reply) |
| 342 | var = False |
| 343 | elif not user_input: |
| 344 | if enabled_by_default: |
| 345 | print(yes_reply) |
| 346 | var = True |
| 347 | else: |
| 348 | print(no_reply) |
| 349 | var = False |
| 350 | else: |
| 351 | print('Invalid selection: %s' % user_input_origin) |
| 352 | return var |
| 353 | |
| 354 | |
| 355 | def set_build_var(environ_cp, var_name, query_item, option_name, |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 356 | enabled_by_default, bazel_config_name=None): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 357 | """Set if query_item will be enabled for the build. |
| 358 | |
| 359 | Ask user if query_item will be enabled. Default is used if no input is given. |
| 360 | Set subprocess environment variable and write to .bazelrc if enabled. |
| 361 | |
| 362 | Args: |
| 363 | environ_cp: copy of the os.environ. |
| 364 | var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". |
| 365 | query_item: string for feature related to the variable, e.g. "Hadoop File |
| 366 | System". |
| 367 | option_name: string for option to define in .bazelrc. |
| 368 | enabled_by_default: boolean for default behavior. |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 369 | bazel_config_name: Name for Bazel --config argument to enable build feature. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 370 | """ |
| 371 | |
| 372 | var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default))) |
| 373 | environ_cp[var_name] = var |
gracehoney | 98ffa7b | 2018-01-29 11:15:32 -0800 | [diff] [blame] | 374 | if var == '1': |
| 375 | write_to_bazelrc('build --define %s=true' % option_name) |
| 376 | elif bazel_config_name is not None: |
| 377 | # TODO(mikecase): Migrate all users of configure.py to use --config Bazel |
| 378 | # options and not to set build configs through environment variables. |
| 379 | write_to_bazelrc('build:%s --define %s=true' |
| 380 | % (bazel_config_name, option_name)) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 381 | |
| 382 | |
| 383 | def set_action_env_var(environ_cp, |
| 384 | var_name, |
| 385 | query_item, |
| 386 | enabled_by_default, |
| 387 | question=None, |
| 388 | yes_reply=None, |
| 389 | no_reply=None): |
| 390 | """Set boolean action_env variable. |
| 391 | |
| 392 | Ask user if query_item will be enabled. Default is used if no input is given. |
| 393 | Set environment variable and write to .bazelrc. |
| 394 | |
| 395 | Args: |
| 396 | environ_cp: copy of the os.environ. |
| 397 | var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". |
| 398 | query_item: string for feature related to the variable, e.g. "Hadoop File |
| 399 | System". |
| 400 | enabled_by_default: boolean for default behavior. |
| 401 | question: optional string for how to ask for user input. |
elilienstein | 6b23b78 | 2018-01-29 15:27:57 -0800 | [diff] [blame] | 402 | yes_reply: optional string for reply when feature is enabled. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 403 | no_reply: optional string for reply when feature is disabled. |
| 404 | """ |
| 405 | var = int( |
| 406 | get_var(environ_cp, var_name, query_item, enabled_by_default, question, |
| 407 | yes_reply, no_reply)) |
| 408 | |
| 409 | write_action_env_to_bazelrc(var_name, var) |
| 410 | environ_cp[var_name] = str(var) |
| 411 | |
| 412 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 413 | def convert_version_to_int(version): |
| 414 | """Convert a version number to a integer that can be used to compare. |
| 415 | |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 416 | Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The |
| 417 | 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored. |
| 418 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 419 | Args: |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 420 | version: a version to be converted |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 421 | |
| 422 | Returns: |
| 423 | An integer if converted successfully, otherwise return None. |
| 424 | """ |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 425 | version = version.split('-')[0] |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 426 | version_segments = version.split('.') |
| 427 | for seg in version_segments: |
| 428 | if not seg.isdigit(): |
| 429 | return None |
gracehoney | 98ffa7b | 2018-01-29 11:15:32 -0800 | [diff] [blame] | 430 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 431 | version_str = ''.join(['%03d' % int(seg) for seg in version_segments]) |
| 432 | return int(version_str) |
| 433 | |
| 434 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 435 | def check_bazel_version(min_version): |
Sami Kama | 825e7a3 | 2018-01-19 22:58:50 +0000 | [diff] [blame] | 436 | """Check installed bazel version is at least min_version. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 437 | |
| 438 | Args: |
| 439 | min_version: string for minimum bazel version. |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 440 | |
| 441 | Returns: |
| 442 | The bazel version detected. |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 443 | """ |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 444 | if which('bazel') is None: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 445 | print('Cannot find bazel. Please install bazel.') |
| 446 | sys.exit(0) |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 447 | curr_version = run_shell(['bazel', '--batch', 'version']) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 448 | |
| 449 | for line in curr_version.split('\n'): |
| 450 | if 'Build label: ' in line: |
| 451 | curr_version = line.split('Build label: ')[1] |
| 452 | break |
| 453 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 454 | min_version_int = convert_version_to_int(min_version) |
| 455 | curr_version_int = convert_version_to_int(curr_version) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 456 | |
| 457 | # Check if current bazel version can be detected properly. |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 458 | if not curr_version_int: |
| 459 | print('WARNING: current bazel installation is not a release version.') |
| 460 | print('Make sure you are running at least bazel %s' % min_version) |
| 461 | return curr_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 462 | |
Michael Case | d94271a | 2017-08-22 17:26:52 -0700 | [diff] [blame] | 463 | print('You have bazel %s installed.' % curr_version) |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 464 | |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 465 | if curr_version_int < min_version_int: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 466 | print('Please upgrade your bazel installation to version %s or higher to ' |
| 467 | 'build TensorFlow!' % min_version) |
| 468 | sys.exit(0) |
A. Unique TensorFlower | 6252d29 | 2017-08-04 00:52:34 -0700 | [diff] [blame] | 469 | return curr_version |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 470 | |
| 471 | |
| 472 | def set_cc_opt_flags(environ_cp): |
| 473 | """Set up architecture-dependent optimization flags. |
| 474 | |
| 475 | Also append CC optimization flags to bazel.rc.. |
| 476 | |
| 477 | Args: |
| 478 | environ_cp: copy of the os.environ. |
| 479 | """ |
| 480 | if is_ppc64le(): |
| 481 | # gcc on ppc64le does not support -march, use mcpu instead |
| 482 | default_cc_opt_flags = '-mcpu=native' |
| 483 | else: |
| 484 | default_cc_opt_flags = '-march=native' |
| 485 | question = ('Please specify optimization flags to use during compilation when' |
| 486 | ' bazel option "--config=opt" is specified [Default is %s]: ' |
| 487 | ) % default_cc_opt_flags |
| 488 | cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS', |
| 489 | question, default_cc_opt_flags) |
| 490 | for opt in cc_opt_flags.split(): |
Michael Case | 0017742 | 2017-11-10 13:14:03 -0800 | [diff] [blame] | 491 | write_to_bazelrc('build:opt --copt=%s' % opt) |
| 492 | # It should be safe on the same build host. |
Michael Case | ec19c46 | 2018-02-20 20:45:33 -0800 | [diff] [blame] | 493 | if not is_ppc64le(): |
| 494 | write_to_bazelrc('build:opt --host_copt=-march=native') |
Michael Case | bb3355d | 2017-11-09 08:46:31 -0800 | [diff] [blame] | 495 | write_to_bazelrc('build:opt --define with_default_optimizations=true') |
Michael Case | 0017742 | 2017-11-10 13:14:03 -0800 | [diff] [blame] | 496 | # TODO(mikecase): Remove these default defines once we are able to get |
| 497 | # TF Lite targets building without them. |
Andrew Selle | 0b15439 | 2017-11-10 10:35:35 -0800 | [diff] [blame] | 498 | write_to_bazelrc('build --copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK') |
| 499 | write_to_bazelrc('build --host_copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 500 | |
| 501 | |
| 502 | def set_tf_cuda_clang(environ_cp): |
| 503 | """set TF_CUDA_CLANG action_env. |
| 504 | |
| 505 | Args: |
| 506 | environ_cp: copy of the os.environ. |
| 507 | """ |
| 508 | question = 'Do you want to use clang as CUDA compiler?' |
| 509 | yes_reply = 'Clang will be used as CUDA compiler.' |
| 510 | no_reply = 'nvcc will be used as CUDA compiler.' |
| 511 | set_action_env_var( |
| 512 | environ_cp, |
| 513 | 'TF_CUDA_CLANG', |
| 514 | None, |
| 515 | False, |
| 516 | question=question, |
| 517 | yes_reply=yes_reply, |
| 518 | no_reply=no_reply) |
| 519 | |
| 520 | |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 521 | def set_tf_download_clang(environ_cp): |
| 522 | """Set TF_DOWNLOAD_CLANG action_env.""" |
| 523 | question = 'Do you want to download a fresh release of clang? (Experimental)' |
| 524 | yes_reply = 'Clang will be downloaded and used to compile tensorflow.' |
| 525 | no_reply = 'Clang will not be downloaded.' |
| 526 | set_action_env_var( |
| 527 | environ_cp, |
| 528 | 'TF_DOWNLOAD_CLANG', |
| 529 | None, |
| 530 | False, |
| 531 | question=question, |
| 532 | yes_reply=yes_reply, |
| 533 | no_reply=no_reply) |
| 534 | |
| 535 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 536 | def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var, |
| 537 | var_default): |
| 538 | """Get var_name either from env, or user or default. |
| 539 | |
| 540 | If var_name has been set as environment variable, use the preset value, else |
| 541 | ask for user input. If no input is provided, the default is used. |
| 542 | |
| 543 | Args: |
| 544 | environ_cp: copy of the os.environ. |
| 545 | var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". |
| 546 | ask_for_var: string for how to ask for user input. |
| 547 | var_default: default value string. |
| 548 | |
| 549 | Returns: |
| 550 | string value for var_name |
| 551 | """ |
| 552 | var = environ_cp.get(var_name) |
| 553 | if not var: |
| 554 | var = get_input(ask_for_var) |
Michael Case | d94271a | 2017-08-22 17:26:52 -0700 | [diff] [blame] | 555 | print('\n') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 556 | if not var: |
| 557 | var = var_default |
| 558 | return var |
| 559 | |
| 560 | |
| 561 | def set_clang_cuda_compiler_path(environ_cp): |
| 562 | """Set CLANG_CUDA_COMPILER_PATH.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 563 | default_clang_path = which('clang') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 564 | ask_clang_path = ('Please specify which clang should be used as device and ' |
| 565 | 'host compiler. [Default is %s]: ') % default_clang_path |
| 566 | |
| 567 | while True: |
| 568 | clang_cuda_compiler_path = get_from_env_or_user_or_default( |
| 569 | environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path, |
| 570 | default_clang_path) |
| 571 | if os.path.exists(clang_cuda_compiler_path): |
| 572 | break |
| 573 | |
| 574 | # Reset and retry |
| 575 | print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path) |
| 576 | environ_cp['CLANG_CUDA_COMPILER_PATH'] = '' |
| 577 | |
| 578 | # Set CLANG_CUDA_COMPILER_PATH |
| 579 | environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path |
| 580 | write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH', |
| 581 | clang_cuda_compiler_path) |
| 582 | |
| 583 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 584 | def prompt_loop_or_load_from_env( |
| 585 | environ_cp, |
| 586 | var_name, |
| 587 | var_default, |
| 588 | ask_for_var, |
| 589 | check_success, |
| 590 | error_msg, |
| 591 | suppress_default_error=False, |
| 592 | n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS |
| 593 | ): |
| 594 | """Loop over user prompts for an ENV param until receiving a valid response. |
| 595 | |
| 596 | For the env param var_name, read from the environment or verify user input |
| 597 | until receiving valid input. When done, set var_name in the environ_cp to its |
| 598 | new value. |
| 599 | |
| 600 | Args: |
| 601 | environ_cp: (Dict) copy of the os.environ. |
| 602 | var_name: (String) string for name of environment variable, e.g. "TF_MYVAR". |
| 603 | var_default: (String) default value string. |
| 604 | ask_for_var: (String) string for how to ask for user input. |
| 605 | check_success: (Function) function that takes one argument and returns a |
| 606 | boolean. Should return True if the value provided is considered valid. May |
| 607 | contain a complex error message if error_msg does not provide enough |
| 608 | information. In that case, set suppress_default_error to True. |
| 609 | error_msg: (String) String with one and only one '%s'. Formatted with each |
| 610 | invalid response upon check_success(input) failure. |
| 611 | suppress_default_error: (Bool) Suppress the above error message in favor of |
| 612 | one from the check_success function. |
| 613 | n_ask_attempts: (Integer) Number of times to query for valid input before |
| 614 | raising an error and quitting. |
| 615 | |
| 616 | Returns: |
| 617 | [String] The value of var_name after querying for input. |
| 618 | |
| 619 | Raises: |
| 620 | UserInputError: if a query has been attempted n_ask_attempts times without |
Shanqing Cai | 805c17c | 2018-01-04 11:04:32 -0500 | [diff] [blame] | 621 | success, assume that the user has made a scripting error, and will |
| 622 | continue to provide invalid input. Raise the error to avoid infinitely |
| 623 | looping. |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 624 | """ |
| 625 | default = environ_cp.get(var_name) or var_default |
| 626 | full_query = '%s [Default is %s]: ' % ( |
| 627 | ask_for_var, |
| 628 | default, |
| 629 | ) |
| 630 | |
| 631 | for _ in range(n_ask_attempts): |
| 632 | val = get_from_env_or_user_or_default(environ_cp, |
| 633 | var_name, |
| 634 | full_query, |
| 635 | default) |
| 636 | if check_success(val): |
| 637 | break |
| 638 | if not suppress_default_error: |
| 639 | print(error_msg % val) |
| 640 | environ_cp[var_name] = '' |
| 641 | else: |
| 642 | raise UserInputError('Invalid %s setting was provided %d times in a row. ' |
| 643 | 'Assuming to be a scripting mistake.' % |
| 644 | (var_name, n_ask_attempts)) |
| 645 | |
| 646 | environ_cp[var_name] = val |
| 647 | return val |
| 648 | |
| 649 | |
| 650 | def create_android_ndk_rule(environ_cp): |
| 651 | """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule.""" |
| 652 | if is_windows() or is_cygwin(): |
| 653 | default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' % |
| 654 | environ_cp['APPDATA']) |
| 655 | elif is_macos(): |
| 656 | default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 657 | else: |
| 658 | default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 659 | |
| 660 | def valid_ndk_path(path): |
| 661 | return (os.path.exists(path) and |
| 662 | os.path.exists(os.path.join(path, 'source.properties'))) |
| 663 | |
| 664 | android_ndk_home_path = prompt_loop_or_load_from_env( |
| 665 | environ_cp, |
| 666 | var_name='ANDROID_NDK_HOME', |
| 667 | var_default=default_ndk_path, |
| 668 | ask_for_var='Please specify the home path of the Android NDK to use.', |
| 669 | check_success=valid_ndk_path, |
| 670 | error_msg=('The path %s or its child file "source.properties" ' |
| 671 | 'does not exist.') |
| 672 | ) |
| 673 | |
| 674 | write_android_ndk_workspace_rule(android_ndk_home_path) |
| 675 | |
| 676 | |
| 677 | def create_android_sdk_rule(environ_cp): |
| 678 | """Set Android variables and write Android SDK WORKSPACE rule.""" |
| 679 | if is_windows() or is_cygwin(): |
| 680 | default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA']) |
| 681 | elif is_macos(): |
| 682 | default_sdk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME'] |
| 683 | else: |
| 684 | default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME'] |
| 685 | |
| 686 | def valid_sdk_path(path): |
| 687 | return (os.path.exists(path) and |
| 688 | os.path.exists(os.path.join(path, 'platforms')) and |
| 689 | os.path.exists(os.path.join(path, 'build-tools'))) |
| 690 | |
| 691 | android_sdk_home_path = prompt_loop_or_load_from_env( |
| 692 | environ_cp, |
| 693 | var_name='ANDROID_SDK_HOME', |
| 694 | var_default=default_sdk_path, |
| 695 | ask_for_var='Please specify the home path of the Android SDK to use.', |
| 696 | check_success=valid_sdk_path, |
| 697 | error_msg=('Either %s does not exist, or it does not contain the ' |
| 698 | 'subdirectories "platforms" and "build-tools".')) |
| 699 | |
| 700 | platforms = os.path.join(android_sdk_home_path, 'platforms') |
| 701 | api_levels = sorted(os.listdir(platforms)) |
| 702 | api_levels = [x.replace('android-', '') for x in api_levels] |
| 703 | |
| 704 | def valid_api_level(api_level): |
| 705 | return os.path.exists(os.path.join(android_sdk_home_path, |
| 706 | 'platforms', |
| 707 | 'android-' + api_level)) |
| 708 | |
| 709 | android_api_level = prompt_loop_or_load_from_env( |
| 710 | environ_cp, |
| 711 | var_name='ANDROID_API_LEVEL', |
| 712 | var_default=api_levels[-1], |
| 713 | ask_for_var=('Please specify the Android SDK API level to use. ' |
| 714 | '[Available levels: %s]') % api_levels, |
| 715 | check_success=valid_api_level, |
| 716 | error_msg='Android-%s is not present in the SDK path.') |
| 717 | |
| 718 | build_tools = os.path.join(android_sdk_home_path, 'build-tools') |
| 719 | versions = sorted(os.listdir(build_tools)) |
| 720 | |
| 721 | def valid_build_tools(version): |
| 722 | return os.path.exists(os.path.join(android_sdk_home_path, |
| 723 | 'build-tools', |
| 724 | version)) |
| 725 | |
| 726 | android_build_tools_version = prompt_loop_or_load_from_env( |
| 727 | environ_cp, |
| 728 | var_name='ANDROID_BUILD_TOOLS_VERSION', |
| 729 | var_default=versions[-1], |
| 730 | ask_for_var=('Please specify an Android build tools version to use. ' |
| 731 | '[Available versions: %s]') % versions, |
| 732 | check_success=valid_build_tools, |
| 733 | error_msg=('The selected SDK does not have build-tools version %s ' |
| 734 | 'available.')) |
| 735 | |
| 736 | write_android_sdk_workspace_rule(android_sdk_home_path, |
| 737 | android_build_tools_version, |
| 738 | android_api_level) |
| 739 | |
| 740 | |
| 741 | def write_android_sdk_workspace_rule(android_sdk_home_path, |
| 742 | android_build_tools_version, |
| 743 | android_api_level): |
| 744 | print('Writing android_sdk_workspace rule.\n') |
| 745 | with open(_TF_WORKSPACE, 'a') as f: |
| 746 | f.write(""" |
| 747 | android_sdk_repository( |
| 748 | name="androidsdk", |
| 749 | api_level=%s, |
| 750 | path="%s", |
| 751 | build_tools_version="%s")\n |
| 752 | """ % (android_api_level, android_sdk_home_path, android_build_tools_version)) |
| 753 | |
| 754 | |
| 755 | def write_android_ndk_workspace_rule(android_ndk_home_path): |
| 756 | print('Writing android_ndk_workspace rule.') |
| 757 | ndk_api_level = check_ndk_level(android_ndk_home_path) |
| 758 | if int(ndk_api_level) not in _SUPPORTED_ANDROID_NDK_VERSIONS: |
| 759 | print('WARNING: The API level of the NDK in %s is %s, which is not ' |
| 760 | 'supported by Bazel (officially supported versions: %s). Please use ' |
| 761 | 'another version. Compiling Android targets may result in confusing ' |
| 762 | 'errors.\n' % (android_ndk_home_path, ndk_api_level, |
| 763 | _SUPPORTED_ANDROID_NDK_VERSIONS)) |
| 764 | with open(_TF_WORKSPACE, 'a') as f: |
| 765 | f.write(""" |
| 766 | android_ndk_repository( |
| 767 | name="androidndk", |
| 768 | path="%s", |
| 769 | api_level=%s)\n |
| 770 | """ % (android_ndk_home_path, ndk_api_level)) |
| 771 | |
| 772 | |
| 773 | def check_ndk_level(android_ndk_home_path): |
| 774 | """Check the revision number of an Android NDK path.""" |
| 775 | properties_path = '%s/source.properties' % android_ndk_home_path |
| 776 | if is_windows() or is_cygwin(): |
| 777 | properties_path = cygpath(properties_path) |
| 778 | with open(properties_path, 'r') as f: |
| 779 | filedata = f.read() |
| 780 | |
| 781 | revision = re.search(r'Pkg.Revision = (\d+)', filedata) |
| 782 | if revision: |
| 783 | return revision.group(1) |
| 784 | return None |
| 785 | |
| 786 | |
| 787 | def workspace_has_any_android_rule(): |
| 788 | """Check the WORKSPACE for existing android_*_repository rules.""" |
| 789 | with open(_TF_WORKSPACE, 'r') as f: |
| 790 | workspace = f.read() |
| 791 | has_any_rule = re.search(r'^android_[ns]dk_repository', |
| 792 | workspace, |
| 793 | re.MULTILINE) |
| 794 | return has_any_rule |
| 795 | |
| 796 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 797 | def set_gcc_host_compiler_path(environ_cp): |
| 798 | """Set GCC_HOST_COMPILER_PATH.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 799 | default_gcc_host_compiler_path = which('gcc') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 800 | cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH') |
| 801 | |
| 802 | if os.path.islink(cuda_bin_symlink): |
| 803 | # os.readlink is only available in linux |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 804 | default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 805 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 806 | gcc_host_compiler_path = prompt_loop_or_load_from_env( |
| 807 | environ_cp, |
| 808 | var_name='GCC_HOST_COMPILER_PATH', |
| 809 | var_default=default_gcc_host_compiler_path, |
| 810 | ask_for_var= |
| 811 | 'Please specify which gcc should be used by nvcc as the host compiler.', |
| 812 | check_success=os.path.exists, |
| 813 | error_msg='Invalid gcc path. %s cannot be found.', |
| 814 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 815 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 816 | write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path) |
| 817 | |
| 818 | |
Gunhan Gulsoy | 5dbd3d7 | 2018-02-08 10:45:37 -0800 | [diff] [blame] | 819 | def reformat_version_sequence(version_str, sequence_count): |
| 820 | """Reformat the version string to have the given number of sequences. |
| 821 | |
| 822 | For example: |
| 823 | Given (7, 2) -> 7.0 |
| 824 | (7.0.1, 2) -> 7.0 |
| 825 | (5, 1) -> 5 |
| 826 | (5.0.3.2, 1) -> 5 |
| 827 | |
| 828 | Args: |
| 829 | version_str: String, the version string. |
| 830 | sequence_count: int, an integer. |
| 831 | Returns: |
| 832 | string, reformatted version string. |
| 833 | """ |
| 834 | v = version_str.split('.') |
| 835 | if len(v) < sequence_count: |
| 836 | v = v + (['0'] * (sequence_count - len(v))) |
| 837 | |
| 838 | return '.'.join(v[:sequence_count]) |
| 839 | |
| 840 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 841 | def set_tf_cuda_version(environ_cp): |
| 842 | """Set CUDA_TOOLKIT_PATH and TF_CUDA_VERSION.""" |
| 843 | ask_cuda_version = ( |
| 844 | 'Please specify the CUDA SDK version you want to use, ' |
| 845 | 'e.g. 7.0. [Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION |
| 846 | |
Austin Anderson | f9a88f8 | 2017-12-13 11:49:40 -0800 | [diff] [blame] | 847 | for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 848 | # Configure the Cuda SDK version to use. |
| 849 | tf_cuda_version = get_from_env_or_user_or_default( |
| 850 | environ_cp, 'TF_CUDA_VERSION', ask_cuda_version, _DEFAULT_CUDA_VERSION) |
Gunhan Gulsoy | 5dbd3d7 | 2018-02-08 10:45:37 -0800 | [diff] [blame] | 851 | tf_cuda_version = reformat_version_sequence(str(tf_cuda_version), 2) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 852 | |
| 853 | # Find out where the CUDA toolkit is installed |
| 854 | default_cuda_path = _DEFAULT_CUDA_PATH |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 855 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 856 | default_cuda_path = cygpath( |
| 857 | environ_cp.get('CUDA_PATH', _DEFAULT_CUDA_PATH_WIN)) |
| 858 | elif is_linux(): |
| 859 | # If the default doesn't exist, try an alternative default. |
| 860 | if (not os.path.exists(default_cuda_path) |
| 861 | ) and os.path.exists(_DEFAULT_CUDA_PATH_LINUX): |
| 862 | default_cuda_path = _DEFAULT_CUDA_PATH_LINUX |
| 863 | ask_cuda_path = ('Please specify the location where CUDA %s toolkit is' |
| 864 | ' installed. Refer to README.md for more details. ' |
| 865 | '[Default is %s]: ') % (tf_cuda_version, default_cuda_path) |
| 866 | cuda_toolkit_path = get_from_env_or_user_or_default( |
| 867 | environ_cp, 'CUDA_TOOLKIT_PATH', ask_cuda_path, default_cuda_path) |
| 868 | |
| 869 | if is_windows(): |
| 870 | cuda_rt_lib_path = 'lib/x64/cudart.lib' |
| 871 | elif is_linux(): |
| 872 | cuda_rt_lib_path = 'lib64/libcudart.so.%s' % tf_cuda_version |
| 873 | elif is_macos(): |
| 874 | cuda_rt_lib_path = 'lib/libcudart.%s.dylib' % tf_cuda_version |
| 875 | |
| 876 | cuda_toolkit_path_full = os.path.join(cuda_toolkit_path, cuda_rt_lib_path) |
| 877 | if os.path.exists(cuda_toolkit_path_full): |
| 878 | break |
| 879 | |
| 880 | # Reset and retry |
| 881 | print('Invalid path to CUDA %s toolkit. %s cannot be found' % |
| 882 | (tf_cuda_version, cuda_toolkit_path_full)) |
| 883 | environ_cp['TF_CUDA_VERSION'] = '' |
| 884 | environ_cp['CUDA_TOOLKIT_PATH'] = '' |
| 885 | |
Austin Anderson | f9a88f8 | 2017-12-13 11:49:40 -0800 | [diff] [blame] | 886 | else: |
| 887 | raise UserInputError('Invalid TF_CUDA_SETTING setting was provided %d ' |
| 888 | 'times in a row. Assuming to be a scripting mistake.' % |
| 889 | _DEFAULT_PROMPT_ASK_ATTEMPTS) |
| 890 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 891 | # Set CUDA_TOOLKIT_PATH and TF_CUDA_VERSION |
| 892 | environ_cp['CUDA_TOOLKIT_PATH'] = cuda_toolkit_path |
| 893 | write_action_env_to_bazelrc('CUDA_TOOLKIT_PATH', cuda_toolkit_path) |
| 894 | environ_cp['TF_CUDA_VERSION'] = tf_cuda_version |
| 895 | write_action_env_to_bazelrc('TF_CUDA_VERSION', tf_cuda_version) |
| 896 | |
| 897 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 898 | def set_tf_cudnn_version(environ_cp): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 899 | """Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION.""" |
| 900 | ask_cudnn_version = ( |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 901 | 'Please specify the cuDNN version you want to use. ' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 902 | '[Leave empty to default to cuDNN %s.0]: ') % _DEFAULT_CUDNN_VERSION |
| 903 | |
Austin Anderson | f9a88f8 | 2017-12-13 11:49:40 -0800 | [diff] [blame] | 904 | for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 905 | tf_cudnn_version = get_from_env_or_user_or_default( |
| 906 | environ_cp, 'TF_CUDNN_VERSION', ask_cudnn_version, |
| 907 | _DEFAULT_CUDNN_VERSION) |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 908 | tf_cudnn_version = reformat_version_sequence(str(tf_cudnn_version), 1) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 909 | |
| 910 | default_cudnn_path = environ_cp.get('CUDA_TOOLKIT_PATH') |
| 911 | ask_cudnn_path = (r'Please specify the location where cuDNN %s library is ' |
| 912 | 'installed. Refer to README.md for more details. [Default' |
| 913 | ' is %s]:') % (tf_cudnn_version, default_cudnn_path) |
| 914 | cudnn_install_path = get_from_env_or_user_or_default( |
| 915 | environ_cp, 'CUDNN_INSTALL_PATH', ask_cudnn_path, default_cudnn_path) |
| 916 | |
| 917 | # Result returned from "read" will be used unexpanded. That make "~" |
| 918 | # unusable. Going through one more level of expansion to handle that. |
| 919 | cudnn_install_path = os.path.realpath( |
| 920 | os.path.expanduser(cudnn_install_path)) |
Martin Wicke | d57572e | 2017-09-02 19:21:45 -0700 | [diff] [blame] | 921 | if is_windows() or is_cygwin(): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 922 | cudnn_install_path = cygpath(cudnn_install_path) |
| 923 | |
| 924 | if is_windows(): |
| 925 | cuda_dnn_lib_path = 'lib/x64/cudnn.lib' |
| 926 | cuda_dnn_lib_alt_path = 'lib/x64/cudnn.lib' |
| 927 | elif is_linux(): |
| 928 | cuda_dnn_lib_path = 'lib64/libcudnn.so.%s' % tf_cudnn_version |
| 929 | cuda_dnn_lib_alt_path = 'libcudnn.so.%s' % tf_cudnn_version |
| 930 | elif is_macos(): |
| 931 | cuda_dnn_lib_path = 'lib/libcudnn.%s.dylib' % tf_cudnn_version |
| 932 | cuda_dnn_lib_alt_path = 'libcudnn.%s.dylib' % tf_cudnn_version |
| 933 | |
| 934 | cuda_dnn_lib_path_full = os.path.join(cudnn_install_path, cuda_dnn_lib_path) |
| 935 | cuda_dnn_lib_alt_path_full = os.path.join(cudnn_install_path, |
| 936 | cuda_dnn_lib_alt_path) |
| 937 | if os.path.exists(cuda_dnn_lib_path_full) or os.path.exists( |
| 938 | cuda_dnn_lib_alt_path_full): |
| 939 | break |
| 940 | |
| 941 | # Try another alternative for Linux |
| 942 | if is_linux(): |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 943 | ldconfig_bin = which('ldconfig') or '/sbin/ldconfig' |
| 944 | cudnn_path_from_ldconfig = run_shell([ldconfig_bin, '-p']) |
| 945 | cudnn_path_from_ldconfig = re.search('.*libcudnn.so .* => (.*)', |
A. Unique TensorFlower | e722358 | 2017-09-06 17:57:04 -0700 | [diff] [blame] | 946 | cudnn_path_from_ldconfig) |
| 947 | if cudnn_path_from_ldconfig: |
| 948 | cudnn_path_from_ldconfig = cudnn_path_from_ldconfig.group(1) |
| 949 | if os.path.exists('%s.%s' % (cudnn_path_from_ldconfig, |
| 950 | tf_cudnn_version)): |
| 951 | cudnn_install_path = os.path.dirname(cudnn_path_from_ldconfig) |
| 952 | break |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 953 | |
| 954 | # Reset and Retry |
| 955 | print( |
| 956 | 'Invalid path to cuDNN %s toolkit. None of the following files can be ' |
| 957 | 'found:' % tf_cudnn_version) |
| 958 | print(cuda_dnn_lib_path_full) |
| 959 | print(cuda_dnn_lib_alt_path_full) |
| 960 | if is_linux(): |
| 961 | print('%s.%s' % (cudnn_path_from_ldconfig, tf_cudnn_version)) |
| 962 | |
| 963 | environ_cp['TF_CUDNN_VERSION'] = '' |
Austin Anderson | f9a88f8 | 2017-12-13 11:49:40 -0800 | [diff] [blame] | 964 | else: |
| 965 | raise UserInputError('Invalid TF_CUDNN setting was provided %d ' |
| 966 | 'times in a row. Assuming to be a scripting mistake.' % |
| 967 | _DEFAULT_PROMPT_ASK_ATTEMPTS) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 968 | |
| 969 | # Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION |
| 970 | environ_cp['CUDNN_INSTALL_PATH'] = cudnn_install_path |
| 971 | write_action_env_to_bazelrc('CUDNN_INSTALL_PATH', cudnn_install_path) |
| 972 | environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version |
| 973 | write_action_env_to_bazelrc('TF_CUDNN_VERSION', tf_cudnn_version) |
| 974 | |
| 975 | |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 976 | def set_tf_tensorrt_install_path(environ_cp): |
| 977 | """Set TENSORRT_INSTALL_PATH and TF_TENSORRT_VERSION. |
| 978 | |
| 979 | Adapted from code contributed by Sami Kama (https://github.com/samikama). |
| 980 | |
| 981 | Args: |
| 982 | environ_cp: copy of the os.environ. |
| 983 | |
| 984 | Raises: |
| 985 | ValueError: if this method was called under non-Linux platform. |
| 986 | UserInputError: if user has provided invalid input multiple times. |
| 987 | """ |
| 988 | if not is_linux(): |
| 989 | raise ValueError('Currently TensorRT is only supported on Linux platform.') |
| 990 | |
| 991 | # Ask user whether to add TensorRT support. |
| 992 | if str(int(get_var( |
| 993 | environ_cp, 'TF_NEED_TENSORRT', 'TensorRT', False))) != '1': |
| 994 | return |
| 995 | |
| 996 | for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): |
| 997 | ask_tensorrt_path = (r'Please specify the location where TensorRT is ' |
| 998 | 'installed. [Default is %s]:') % ( |
| 999 | _DEFAULT_TENSORRT_PATH_LINUX) |
| 1000 | trt_install_path = get_from_env_or_user_or_default( |
| 1001 | environ_cp, 'TENSORRT_INSTALL_PATH', ask_tensorrt_path, |
| 1002 | _DEFAULT_TENSORRT_PATH_LINUX) |
| 1003 | |
| 1004 | # Result returned from "read" will be used unexpanded. That make "~" |
| 1005 | # unusable. Going through one more level of expansion to handle that. |
| 1006 | trt_install_path = os.path.realpath( |
| 1007 | os.path.expanduser(trt_install_path)) |
| 1008 | |
| 1009 | def find_libs(search_path): |
| 1010 | """Search for libnvinfer.so in "search_path".""" |
| 1011 | fl = set() |
| 1012 | if os.path.exists(search_path) and os.path.isdir(search_path): |
| 1013 | fl.update([os.path.realpath(os.path.join(search_path, x)) |
| 1014 | for x in os.listdir(search_path) if 'libnvinfer.so' in x]) |
| 1015 | return fl |
| 1016 | |
| 1017 | possible_files = find_libs(trt_install_path) |
| 1018 | possible_files.update(find_libs(os.path.join(trt_install_path, 'lib'))) |
| 1019 | possible_files.update(find_libs(os.path.join(trt_install_path, 'lib64'))) |
| 1020 | |
| 1021 | def is_compatible(tensorrt_lib, cuda_ver, cudnn_ver): |
| 1022 | """Check the compatibility between tensorrt and cudnn/cudart libraries.""" |
| 1023 | ldd_bin = which('ldd') or '/usr/bin/ldd' |
| 1024 | ldd_out = run_shell([ldd_bin, tensorrt_lib]).split(os.linesep) |
| 1025 | cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$') |
| 1026 | cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$') |
| 1027 | cudnn = None |
| 1028 | cudart = None |
| 1029 | for line in ldd_out: |
| 1030 | if 'libcudnn.so' in line: |
| 1031 | cudnn = cudnn_pattern.search(line) |
| 1032 | elif 'libcudart.so' in line: |
| 1033 | cudart = cuda_pattern.search(line) |
| 1034 | if cudnn and len(cudnn.group(1)): |
| 1035 | cudnn = convert_version_to_int(cudnn.group(1)) |
| 1036 | if cudart and len(cudart.group(1)): |
| 1037 | cudart = convert_version_to_int(cudart.group(1)) |
| 1038 | return (cudnn == cudnn_ver) and (cudart == cuda_ver) |
| 1039 | |
| 1040 | cuda_ver = convert_version_to_int(environ_cp['TF_CUDA_VERSION']) |
| 1041 | cudnn_ver = convert_version_to_int(environ_cp['TF_CUDNN_VERSION']) |
| 1042 | nvinfer_pattern = re.compile('.*libnvinfer.so.?(.*)$') |
| 1043 | highest_ver = [0, None, None] |
| 1044 | |
| 1045 | for lib_file in possible_files: |
| 1046 | if is_compatible(lib_file, cuda_ver, cudnn_ver): |
| 1047 | ver_str = nvinfer_pattern.search(lib_file).group(1) |
| 1048 | ver = convert_version_to_int(ver_str) if len(ver_str) else 0 |
| 1049 | if ver > highest_ver[0]: |
| 1050 | highest_ver = [ver, ver_str, lib_file] |
| 1051 | if highest_ver[1] is not None: |
| 1052 | trt_install_path = os.path.dirname(highest_ver[2]) |
| 1053 | tf_tensorrt_version = highest_ver[1] |
| 1054 | break |
| 1055 | |
| 1056 | # Try another alternative from ldconfig. |
| 1057 | ldconfig_bin = which('ldconfig') or '/sbin/ldconfig' |
| 1058 | ldconfig_output = run_shell([ldconfig_bin, '-p']) |
| 1059 | search_result = re.search( |
| 1060 | '.*libnvinfer.so\\.?([0-9.]*).* => (.*)', ldconfig_output) |
| 1061 | if search_result: |
| 1062 | libnvinfer_path_from_ldconfig = search_result.group(2) |
| 1063 | if os.path.exists(libnvinfer_path_from_ldconfig): |
| 1064 | if is_compatible(libnvinfer_path_from_ldconfig, cuda_ver, cudnn_ver): |
| 1065 | trt_install_path = os.path.dirname(libnvinfer_path_from_ldconfig) |
| 1066 | tf_tensorrt_version = search_result.group(1) |
| 1067 | break |
| 1068 | |
| 1069 | # Reset and Retry |
Sami Kama | 35ba5d8 | 2018-02-12 13:35:23 -0800 | [diff] [blame] | 1070 | if len(possible_files): |
| 1071 | print('TensorRT libraries found in one the following directories', |
| 1072 | 'are not compatible with selected cuda and cudnn installations') |
| 1073 | print(trt_install_path) |
| 1074 | print(os.path.join(trt_install_path, 'lib')) |
| 1075 | print(os.path.join(trt_install_path, 'lib64')) |
| 1076 | if search_result: |
| 1077 | print(libnvinfer_path_from_ldconfig) |
| 1078 | else: |
| 1079 | print('Invalid path to TensorRT. None of the following files can be found:') |
| 1080 | print(trt_install_path) |
| 1081 | print(os.path.join(trt_install_path, 'lib')) |
| 1082 | print(os.path.join(trt_install_path, 'lib64')) |
| 1083 | if search_result: |
| 1084 | print(libnvinfer_path_from_ldconfig) |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 1085 | |
| 1086 | else: |
| 1087 | raise UserInputError('Invalid TF_TENSORRT setting was provided %d ' |
| 1088 | 'times in a row. Assuming to be a scripting mistake.' % |
| 1089 | _DEFAULT_PROMPT_ASK_ATTEMPTS) |
| 1090 | |
| 1091 | # Set TENSORRT_INSTALL_PATH and TF_TENSORRT_VERSION |
| 1092 | environ_cp['TENSORRT_INSTALL_PATH'] = trt_install_path |
| 1093 | write_action_env_to_bazelrc('TENSORRT_INSTALL_PATH', trt_install_path) |
| 1094 | environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version |
| 1095 | write_action_env_to_bazelrc('TF_TENSORRT_VERSION', tf_tensorrt_version) |
| 1096 | |
| 1097 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1098 | def get_native_cuda_compute_capabilities(environ_cp): |
| 1099 | """Get native cuda compute capabilities. |
| 1100 | |
| 1101 | Args: |
| 1102 | environ_cp: copy of the os.environ. |
| 1103 | Returns: |
| 1104 | string of native cuda compute capabilities, separated by comma. |
| 1105 | """ |
| 1106 | device_query_bin = os.path.join( |
| 1107 | environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery') |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1108 | if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK): |
| 1109 | try: |
| 1110 | output = run_shell(device_query_bin).split('\n') |
| 1111 | pattern = re.compile('[0-9]*\\.[0-9]*') |
| 1112 | output = [pattern.search(x) for x in output if 'Capability' in x] |
| 1113 | output = ','.join(x.group() for x in output if x is not None) |
| 1114 | except subprocess.CalledProcessError: |
| 1115 | output = '' |
| 1116 | else: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1117 | output = '' |
| 1118 | return output |
| 1119 | |
| 1120 | |
| 1121 | def set_tf_cuda_compute_capabilities(environ_cp): |
| 1122 | """Set TF_CUDA_COMPUTE_CAPABILITIES.""" |
| 1123 | while True: |
| 1124 | native_cuda_compute_capabilities = get_native_cuda_compute_capabilities( |
| 1125 | environ_cp) |
| 1126 | if not native_cuda_compute_capabilities: |
| 1127 | default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES |
| 1128 | else: |
| 1129 | default_cuda_compute_capabilities = native_cuda_compute_capabilities |
| 1130 | |
| 1131 | ask_cuda_compute_capabilities = ( |
| 1132 | 'Please specify a list of comma-separated ' |
| 1133 | 'Cuda compute capabilities you want to ' |
| 1134 | 'build with.\nYou can find the compute ' |
| 1135 | 'capability of your device at: ' |
| 1136 | 'https://developer.nvidia.com/cuda-gpus.\nPlease' |
| 1137 | ' note that each additional compute ' |
| 1138 | 'capability significantly increases your ' |
| 1139 | 'build time and binary size. [Default is: %s]' % |
| 1140 | default_cuda_compute_capabilities) |
| 1141 | tf_cuda_compute_capabilities = get_from_env_or_user_or_default( |
| 1142 | environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES', |
| 1143 | ask_cuda_compute_capabilities, default_cuda_compute_capabilities) |
| 1144 | # Check whether all capabilities from the input is valid |
| 1145 | all_valid = True |
| 1146 | for compute_capability in tf_cuda_compute_capabilities.split(','): |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1147 | m = re.match('[0-9]+.[0-9]+', compute_capability) |
| 1148 | if not m: |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1149 | print('Invalid compute capability: ' % compute_capability) |
| 1150 | all_valid = False |
Andrew Harp | 6e3e7d1 | 2017-08-21 12:10:44 -0700 | [diff] [blame] | 1151 | else: |
| 1152 | ver = int(m.group(0).split('.')[0]) |
| 1153 | if ver < 3: |
| 1154 | print('Only compute capabilities 3.0 or higher are supported.') |
| 1155 | all_valid = False |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1156 | |
| 1157 | if all_valid: |
| 1158 | break |
| 1159 | |
| 1160 | # Reset and Retry |
| 1161 | environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = '' |
| 1162 | |
| 1163 | # Set TF_CUDA_COMPUTE_CAPABILITIES |
| 1164 | environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities |
| 1165 | write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES', |
| 1166 | tf_cuda_compute_capabilities) |
| 1167 | |
| 1168 | |
| 1169 | def set_other_cuda_vars(environ_cp): |
| 1170 | """Set other CUDA related variables.""" |
| 1171 | if is_windows(): |
| 1172 | # The following three variables are needed for MSVC toolchain configuration |
| 1173 | # in Bazel |
| 1174 | environ_cp['CUDA_PATH'] = environ_cp.get('CUDA_TOOLKIT_PATH') |
| 1175 | environ_cp['CUDA_COMPUTE_CAPABILITIES'] = environ_cp.get( |
| 1176 | 'TF_CUDA_COMPUTE_CAPABILITIES') |
| 1177 | environ_cp['NO_WHOLE_ARCHIVE_OPTION'] = 1 |
| 1178 | write_action_env_to_bazelrc('CUDA_PATH', environ_cp.get('CUDA_PATH')) |
| 1179 | write_action_env_to_bazelrc('CUDA_COMPUTE_CAPABILITIE', |
| 1180 | environ_cp.get('CUDA_COMPUTE_CAPABILITIE')) |
| 1181 | write_action_env_to_bazelrc('NO_WHOLE_ARCHIVE_OPTION', |
| 1182 | environ_cp.get('NO_WHOLE_ARCHIVE_OPTION')) |
| 1183 | write_to_bazelrc('build --config=win-cuda') |
| 1184 | write_to_bazelrc('test --config=win-cuda') |
| 1185 | else: |
| 1186 | # If CUDA is enabled, always use GPU during build and test. |
| 1187 | if environ_cp.get('TF_CUDA_CLANG') == '1': |
| 1188 | write_to_bazelrc('build --config=cuda_clang') |
| 1189 | write_to_bazelrc('test --config=cuda_clang') |
| 1190 | else: |
| 1191 | write_to_bazelrc('build --config=cuda') |
| 1192 | write_to_bazelrc('test --config=cuda') |
| 1193 | |
| 1194 | |
| 1195 | def set_host_cxx_compiler(environ_cp): |
| 1196 | """Set HOST_CXX_COMPILER.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1197 | default_cxx_host_compiler = which('g++') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1198 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1199 | host_cxx_compiler = prompt_loop_or_load_from_env( |
| 1200 | environ_cp, |
| 1201 | var_name='HOST_CXX_COMPILER', |
| 1202 | var_default=default_cxx_host_compiler, |
| 1203 | ask_for_var=('Please specify which C++ compiler should be used as the ' |
| 1204 | 'host C++ compiler.'), |
| 1205 | check_success=os.path.exists, |
| 1206 | error_msg='Invalid C++ compiler path. %s cannot be found.', |
| 1207 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1208 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1209 | write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler) |
| 1210 | |
| 1211 | |
| 1212 | def set_host_c_compiler(environ_cp): |
| 1213 | """Set HOST_C_COMPILER.""" |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1214 | default_c_host_compiler = which('gcc') or '' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1215 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1216 | host_c_compiler = prompt_loop_or_load_from_env( |
| 1217 | environ_cp, |
| 1218 | var_name='HOST_C_COMPILER', |
| 1219 | var_default=default_c_host_compiler, |
Seo Sanghyeon | 16625e9 | 2018-02-23 05:27:40 +0900 | [diff] [blame] | 1220 | ask_for_var=('Please specify which C compiler should be used as the host ' |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1221 | 'C compiler.'), |
| 1222 | check_success=os.path.exists, |
| 1223 | error_msg='Invalid C compiler path. %s cannot be found.', |
| 1224 | ) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1225 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1226 | write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler) |
| 1227 | |
| 1228 | |
| 1229 | def set_computecpp_toolkit_path(environ_cp): |
| 1230 | """Set COMPUTECPP_TOOLKIT_PATH.""" |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1231 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1232 | def toolkit_exists(toolkit_path): |
| 1233 | """Check if a computecpp toolkit path is valid.""" |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1234 | if is_linux(): |
| 1235 | sycl_rt_lib_path = 'lib/libComputeCpp.so' |
| 1236 | else: |
| 1237 | sycl_rt_lib_path = '' |
| 1238 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1239 | sycl_rt_lib_path_full = os.path.join(toolkit_path, |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1240 | sycl_rt_lib_path) |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1241 | exists = os.path.exists(sycl_rt_lib_path_full) |
| 1242 | if not exists: |
| 1243 | print('Invalid SYCL %s library path. %s cannot be found' % |
| 1244 | (_TF_OPENCL_VERSION, sycl_rt_lib_path_full)) |
| 1245 | return exists |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1246 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1247 | computecpp_toolkit_path = prompt_loop_or_load_from_env( |
| 1248 | environ_cp, |
| 1249 | var_name='COMPUTECPP_TOOLKIT_PATH', |
| 1250 | var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH, |
| 1251 | ask_for_var=( |
| 1252 | 'Please specify the location where ComputeCpp for SYCL %s is ' |
| 1253 | 'installed.' % _TF_OPENCL_VERSION), |
| 1254 | check_success=toolkit_exists, |
| 1255 | error_msg='Invalid SYCL compiler path. %s cannot be found.', |
| 1256 | suppress_default_error=True) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1257 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1258 | write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH', |
| 1259 | computecpp_toolkit_path) |
| 1260 | |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1261 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1262 | def set_trisycl_include_dir(environ_cp): |
Shanqing Cai | 805c17c | 2018-01-04 11:04:32 -0500 | [diff] [blame] | 1263 | """Set TRISYCL_INCLUDE_DIR.""" |
Raghu Krishnamoorthi | d29b712 | 2018-01-05 17:18:52 -0800 | [diff] [blame] | 1264 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1265 | ask_trisycl_include_dir = ('Please specify the location of the triSYCL ' |
| 1266 | 'include directory. (Use --config=sycl_trisycl ' |
| 1267 | 'when building with Bazel) ' |
| 1268 | '[Default is %s]: ' |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1269 | ) % (_DEFAULT_TRISYCL_INCLUDE_DIR) |
Raghu Krishnamoorthi | d29b712 | 2018-01-05 17:18:52 -0800 | [diff] [blame] | 1270 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1271 | while True: |
| 1272 | trisycl_include_dir = get_from_env_or_user_or_default( |
Shanqing Cai | 805c17c | 2018-01-04 11:04:32 -0500 | [diff] [blame] | 1273 | environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir, |
| 1274 | _DEFAULT_TRISYCL_INCLUDE_DIR) |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1275 | if os.path.exists(trisycl_include_dir): |
| 1276 | break |
| 1277 | |
| 1278 | print('Invalid triSYCL include directory, %s cannot be found' |
| 1279 | % (trisycl_include_dir)) |
| 1280 | |
| 1281 | # Set TRISYCL_INCLUDE_DIR |
| 1282 | environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir |
| 1283 | write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', |
| 1284 | trisycl_include_dir) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1285 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1286 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1287 | def set_mpi_home(environ_cp): |
| 1288 | """Set MPI_HOME.""" |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1289 | |
Jonathan Hseu | 008910f | 2017-08-25 14:01:05 -0700 | [diff] [blame] | 1290 | default_mpi_home = which('mpirun') or which('mpiexec') or '' |
| 1291 | default_mpi_home = os.path.dirname(os.path.dirname(default_mpi_home)) |
| 1292 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1293 | def valid_mpi_path(mpi_home): |
| 1294 | exists = (os.path.exists(os.path.join(mpi_home, 'include')) and |
| 1295 | os.path.exists(os.path.join(mpi_home, 'lib'))) |
| 1296 | if not exists: |
| 1297 | print('Invalid path to the MPI Toolkit. %s or %s cannot be found' % |
| 1298 | (os.path.join(mpi_home, 'include'), |
| 1299 | os.path.exists(os.path.join(mpi_home, 'lib')))) |
| 1300 | return exists |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1301 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1302 | _ = prompt_loop_or_load_from_env( |
| 1303 | environ_cp, |
| 1304 | var_name='MPI_HOME', |
| 1305 | var_default=default_mpi_home, |
| 1306 | ask_for_var='Please specify the MPI toolkit folder.', |
| 1307 | check_success=valid_mpi_path, |
| 1308 | error_msg='', |
| 1309 | suppress_default_error=True) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1310 | |
| 1311 | |
| 1312 | def set_other_mpi_vars(environ_cp): |
| 1313 | """Set other MPI related variables.""" |
| 1314 | # Link the MPI header files |
| 1315 | mpi_home = environ_cp.get('MPI_HOME') |
| 1316 | symlink_force('%s/include/mpi.h' % mpi_home, 'third_party/mpi/mpi.h') |
| 1317 | |
| 1318 | # Determine if we use OpenMPI or MVAPICH, these require different header files |
| 1319 | # to be included here to make bazel dependency checker happy |
| 1320 | if os.path.exists(os.path.join(mpi_home, 'include/mpi_portable_platform.h')): |
| 1321 | symlink_force( |
| 1322 | os.path.join(mpi_home, 'include/mpi_portable_platform.h'), |
| 1323 | 'third_party/mpi/mpi_portable_platform.h') |
| 1324 | # TODO(gunan): avoid editing files in configure |
| 1325 | sed_in_place('third_party/mpi/mpi.bzl', 'MPI_LIB_IS_OPENMPI=False', |
| 1326 | 'MPI_LIB_IS_OPENMPI=True') |
| 1327 | else: |
| 1328 | # MVAPICH / MPICH |
| 1329 | symlink_force( |
| 1330 | os.path.join(mpi_home, 'include/mpio.h'), 'third_party/mpi/mpio.h') |
| 1331 | symlink_force( |
| 1332 | os.path.join(mpi_home, 'include/mpicxx.h'), 'third_party/mpi/mpicxx.h') |
| 1333 | # TODO(gunan): avoid editing files in configure |
| 1334 | sed_in_place('third_party/mpi/mpi.bzl', 'MPI_LIB_IS_OPENMPI=True', |
| 1335 | 'MPI_LIB_IS_OPENMPI=False') |
| 1336 | |
| 1337 | if os.path.exists(os.path.join(mpi_home, 'lib/libmpi.so')): |
| 1338 | symlink_force( |
| 1339 | os.path.join(mpi_home, 'lib/libmpi.so'), 'third_party/mpi/libmpi.so') |
| 1340 | else: |
| 1341 | raise ValueError('Cannot find the MPI library file in %s/lib' % mpi_home) |
| 1342 | |
| 1343 | |
A. Unique TensorFlower | 061c359 | 2017-11-13 14:21:04 -0800 | [diff] [blame] | 1344 | def set_grpc_build_flags(): |
| 1345 | write_to_bazelrc('build --define grpc_no_ares=true') |
| 1346 | |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1347 | |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1348 | def set_windows_build_flags(): |
| 1349 | if is_windows(): |
| 1350 | # The non-monolithic build is not supported yet |
| 1351 | write_to_bazelrc('build --config monolithic') |
| 1352 | # Suppress warning messages |
| 1353 | write_to_bazelrc('build --copt=-w --host_copt=-w') |
| 1354 | # Output more verbose information when something goes wrong |
| 1355 | write_to_bazelrc('build --verbose_failures') |
| 1356 | |
A. Unique TensorFlower | 061c359 | 2017-11-13 14:21:04 -0800 | [diff] [blame] | 1357 | |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1358 | def config_info_line(name, help_text): |
| 1359 | """Helper function to print formatted help text for Bazel config options.""" |
| 1360 | print('\t--config=%-12s\t# %s' % (name, help_text)) |
| 1361 | |
| 1362 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1363 | def main(): |
Michael Case | 671baf0 | 2018-02-21 20:39:21 -0800 | [diff] [blame] | 1364 | parser = argparse.ArgumentParser() |
| 1365 | parser.add_argument("--workspace", |
| 1366 | type=str, |
| 1367 | default=_TF_WORKSPACE_ROOT, |
| 1368 | help="The absolute path to your active Bazel workspace.") |
| 1369 | args = parser.parse_args() |
| 1370 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1371 | # Make a copy of os.environ to be clear when functions and getting and setting |
| 1372 | # environment variables. |
| 1373 | environ_cp = dict(os.environ) |
| 1374 | |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 1375 | check_bazel_version('0.5.4') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1376 | |
Michael Case | 671baf0 | 2018-02-21 20:39:21 -0800 | [diff] [blame] | 1377 | reset_tf_configure_bazelrc(args.workspace) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1378 | cleanup_makefile() |
Gunhan Gulsoy | ed89a2b | 2017-09-19 18:36:26 -0700 | [diff] [blame] | 1379 | setup_python(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1380 | |
| 1381 | if is_windows(): |
Benoit Steiner | 355e25e | 2017-10-24 19:47:46 -0700 | [diff] [blame] | 1382 | environ_cp['TF_NEED_S3'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1383 | environ_cp['TF_NEED_GCP'] = '0' |
| 1384 | environ_cp['TF_NEED_HDFS'] = '0' |
| 1385 | environ_cp['TF_NEED_JEMALLOC'] = '0' |
Yong Tang | 16a2a9f | 2018-01-26 12:45:35 -0800 | [diff] [blame] | 1386 | environ_cp['TF_NEED_KAFKA'] = '0' |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1387 | environ_cp['TF_NEED_OPENCL_SYCL'] = '0' |
| 1388 | environ_cp['TF_NEED_COMPUTECPP'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1389 | environ_cp['TF_NEED_OPENCL'] = '0' |
| 1390 | environ_cp['TF_CUDA_CLANG'] = '0' |
Sami Kama | 825e7a3 | 2018-01-19 22:58:50 +0000 | [diff] [blame] | 1391 | environ_cp['TF_NEED_TENSORRT'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1392 | |
| 1393 | if is_macos(): |
| 1394 | environ_cp['TF_NEED_JEMALLOC'] = '0' |
Sami Kama | 825e7a3 | 2018-01-19 22:58:50 +0000 | [diff] [blame] | 1395 | environ_cp['TF_NEED_TENSORRT'] = '0' |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1396 | |
| 1397 | set_build_var(environ_cp, 'TF_NEED_JEMALLOC', 'jemalloc as malloc', |
| 1398 | 'with_jemalloc', True) |
| 1399 | set_build_var(environ_cp, 'TF_NEED_GCP', 'Google Cloud Platform', |
Benoit Steiner | 355e25e | 2017-10-24 19:47:46 -0700 | [diff] [blame] | 1400 | 'with_gcp_support', True, 'gcp') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1401 | set_build_var(environ_cp, 'TF_NEED_HDFS', 'Hadoop File System', |
Benoit Steiner | 355e25e | 2017-10-24 19:47:46 -0700 | [diff] [blame] | 1402 | 'with_hdfs_support', True, 'hdfs') |
Michael Case | 7e4e336 | 2017-10-09 13:31:15 -0700 | [diff] [blame] | 1403 | set_build_var(environ_cp, 'TF_NEED_S3', 'Amazon S3 File System', |
| 1404 | 'with_s3_support', True, 's3') |
Yong Tang | 16a2a9f | 2018-01-26 12:45:35 -0800 | [diff] [blame] | 1405 | set_build_var(environ_cp, 'TF_NEED_KAFKA', 'Apache Kafka Platform', |
| 1406 | 'with_kafka_support', False, 'kafka') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1407 | set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support', |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 1408 | False, 'xla') |
A. Unique TensorFlower | 28ce1d1 | 2017-08-15 12:08:29 -0700 | [diff] [blame] | 1409 | set_build_var(environ_cp, 'TF_NEED_GDR', 'GDR', 'with_gdr_support', |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 1410 | False, 'gdr') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1411 | set_build_var(environ_cp, 'TF_NEED_VERBS', 'VERBS', 'with_verbs_support', |
Michael Case | 98850a5 | 2017-09-14 13:35:57 -0700 | [diff] [blame] | 1412 | False, 'verbs') |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1413 | |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1414 | set_action_env_var(environ_cp, 'TF_NEED_OPENCL_SYCL', 'OpenCL SYCL', False) |
| 1415 | if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1416 | set_host_cxx_compiler(environ_cp) |
| 1417 | set_host_c_compiler(environ_cp) |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1418 | set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True) |
| 1419 | if environ_cp.get('TF_NEED_COMPUTECPP') == '1': |
| 1420 | set_computecpp_toolkit_path(environ_cp) |
| 1421 | else: |
| 1422 | set_trisycl_include_dir(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1423 | |
| 1424 | set_action_env_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False) |
A. Unique TensorFlower | 24cbb2a | 2017-09-08 07:45:44 -0700 | [diff] [blame] | 1425 | if (environ_cp.get('TF_NEED_CUDA') == '1' and |
| 1426 | 'TF_CUDA_CONFIG_REPO' not in environ_cp): |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1427 | set_tf_cuda_version(environ_cp) |
Yifei Feng | b1d8c59 | 2017-11-22 13:42:21 -0800 | [diff] [blame] | 1428 | set_tf_cudnn_version(environ_cp) |
Guangda Lai | 76f6938 | 2018-01-25 23:59:19 -0800 | [diff] [blame] | 1429 | if is_linux(): |
| 1430 | set_tf_tensorrt_install_path(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1431 | set_tf_cuda_compute_capabilities(environ_cp) |
Ankur Taly | 0e6f39d | 2018-02-16 18:22:55 -0800 | [diff] [blame] | 1432 | if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get( |
| 1433 | 'LD_LIBRARY_PATH') != '1': |
| 1434 | write_action_env_to_bazelrc('LD_LIBRARY_PATH', |
| 1435 | environ_cp.get('LD_LIBRARY_PATH')) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1436 | |
| 1437 | set_tf_cuda_clang(environ_cp) |
| 1438 | if environ_cp.get('TF_CUDA_CLANG') == '1': |
A. Unique TensorFlower | fd29e95 | 2017-12-22 03:07:51 -0800 | [diff] [blame] | 1439 | if not is_windows(): |
| 1440 | # Ask if we want to download clang release while building. |
| 1441 | set_tf_download_clang(environ_cp) |
| 1442 | else: |
| 1443 | # We use bazel's generated crosstool on Windows and there is no |
| 1444 | # way to provide downloaded toolchain for that yet. |
| 1445 | # TODO(ibiryukov): Investigate using clang as a cuda compiler on |
| 1446 | # Windows. |
| 1447 | environ_cp['TF_DOWNLOAD_CLANG'] = '0' |
| 1448 | |
| 1449 | if environ_cp.get('TF_DOWNLOAD_CLANG') != '1': |
| 1450 | # Set up which clang we should use as the cuda / host compiler. |
| 1451 | set_clang_cuda_compiler_path(environ_cp) |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1452 | else: |
| 1453 | # Set up which gcc nvcc should use as the host compiler |
| 1454 | # No need to set this on Windows |
| 1455 | if not is_windows(): |
| 1456 | set_gcc_host_compiler_path(environ_cp) |
| 1457 | set_other_cuda_vars(environ_cp) |
| 1458 | |
| 1459 | set_build_var(environ_cp, 'TF_NEED_MPI', 'MPI', 'with_mpi_support', False) |
| 1460 | if environ_cp.get('TF_NEED_MPI') == '1': |
| 1461 | set_mpi_home(environ_cp) |
| 1462 | set_other_mpi_vars(environ_cp) |
| 1463 | |
A. Unique TensorFlower | 061c359 | 2017-11-13 14:21:04 -0800 | [diff] [blame] | 1464 | set_grpc_build_flags() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1465 | set_cc_opt_flags(environ_cp) |
Dandelion Man? | 90e42f3 | 2017-12-15 18:15:07 -0800 | [diff] [blame] | 1466 | set_windows_build_flags() |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1467 | |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1468 | if workspace_has_any_android_rule(): |
| 1469 | print('The WORKSPACE file has at least one of ["android_sdk_repository", ' |
| 1470 | '"android_ndk_repository"] already set. Will not ask to help ' |
| 1471 | 'configure the WORKSPACE. Please delete the existing rules to ' |
| 1472 | 'activate the helper.\n') |
| 1473 | else: |
| 1474 | if get_var( |
| 1475 | environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', |
| 1476 | False, |
| 1477 | ('Would you like to interactively configure ./WORKSPACE for ' |
| 1478 | 'Android builds?'), |
| 1479 | 'Searching for NDK and SDK installations.', |
| 1480 | 'Not configuring the WORKSPACE for Android builds.'): |
| 1481 | create_android_ndk_rule(environ_cp) |
| 1482 | create_android_sdk_rule(environ_cp) |
| 1483 | |
Michael Case | d31531a | 2018-01-05 14:09:41 -0800 | [diff] [blame] | 1484 | print('Preconfigured Bazel build configs. You can use any of the below by ' |
| 1485 | 'adding "--config=<>" to your build command. See tools/bazel.rc for ' |
| 1486 | 'more details.') |
| 1487 | config_info_line('mkl', 'Build with MKL support.') |
| 1488 | config_info_line('monolithic', 'Config for mostly static monolithic build.') |
Austin Anderson | 6afface | 2017-12-05 11:59:17 -0800 | [diff] [blame] | 1489 | |
A. Unique TensorFlower | 73ea287 | 2017-07-25 13:30:03 -0700 | [diff] [blame] | 1490 | if __name__ == '__main__': |
| 1491 | main() |