blob: a99866610b249df2f250a37958662e31285c21fc [file] [log] [blame]
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""configure script to get build parameters from user."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
Shanqing Cai71445712018-03-12 19:33:52 -070021import argparse
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070022import errno
23import os
24import platform
25import re
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070026import subprocess
27import sys
28
Andrew Sellec9885ea2017-11-06 09:37:03 -080029# pylint: disable=g-import-not-at-top
Jonathan Hseu008910f2017-08-25 14:01:05 -070030try:
31 from shutil import which
32except ImportError:
33 from distutils.spawn import find_executable as which
Andrew Sellec9885ea2017-11-06 09:37:03 -080034# pylint: enable=g-import-not-at-top
Jonathan Hseu008910f2017-08-25 14:01:05 -070035
A. Unique TensorFlower7ed38302019-11-13 11:14:29 -080036_DEFAULT_CUDA_VERSION = '10.1'
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -070037_DEFAULT_CUDNN_VERSION = '7'
Guangda Laifaa93ac2019-10-23 15:07:12 -070038_DEFAULT_TENSORRT_VERSION = '6'
Smit Hinsufe7d1d92018-07-14 13:16:58 -070039_DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0'
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -070040
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070041_TF_OPENCL_VERSION = '1.2'
42_DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp'
Yifei Fengb1d8c592017-11-22 13:42:21 -080043_DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include'
A. Unique TensorFlower82820ef2018-11-12 13:22:13 -080044_SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15, 16, 17, 18]
Austin Anderson6afface2017-12-05 11:59:17 -080045
46_DEFAULT_PROMPT_ASK_ATTEMPTS = 10
47
Shanqing Cai71445712018-03-12 19:33:52 -070048_TF_BAZELRC_FILENAME = '.tf_configure.bazelrc'
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -070049_TF_WORKSPACE_ROOT = ''
50_TF_BAZELRC = ''
A. Unique TensorFlowered297342019-03-15 11:25:28 -070051_TF_CURRENT_BAZEL_VERSION = None
Gunhan Gulsoyaaea5412019-11-29 22:08:00 -080052_TF_MIN_BAZEL_VERSION = '1.0.0'
A. Unique TensorFlowera5f9bcd2019-11-12 07:08:54 -080053_TF_MAX_BAZEL_VERSION = '1.1.0'
Shanqing Cai71445712018-03-12 19:33:52 -070054
Jason Furmanek7c234152018-09-26 04:44:12 +000055NCCL_LIB_PATHS = [
TensorFlower Gardenerea5c5292018-10-01 17:28:26 -070056 'lib64/', 'lib/powerpc64le-linux-gnu/', 'lib/x86_64-linux-gnu/', ''
Jason Furmanek7c234152018-09-26 04:44:12 +000057]
Austin Anderson6afface2017-12-05 11:59:17 -080058
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -070059# List of files to configure when building Bazel on Apple platforms.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -080060APPLE_BAZEL_FILES = [
A. Unique TensorFlower6a059c02019-04-05 14:28:27 -070061 'tensorflow/lite/experimental/ios/BUILD',
A. Unique TensorFlower93e70732019-02-14 16:45:32 -080062 'tensorflow/lite/experimental/objc/BUILD',
YoungSeok Yoon7089aaa2019-11-11 21:36:17 -080063 'tensorflow/lite/experimental/swift/BUILD',
64 'tensorflow/lite/tools/benchmark/experimental/ios/BUILD'
A. Unique TensorFlower93e70732019-02-14 16:45:32 -080065]
66
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -070067# List of files to move when building for iOS.
68IOS_FILES = [
69 'tensorflow/lite/experimental/objc/TensorFlowLiteObjC.podspec',
70 'tensorflow/lite/experimental/swift/TensorFlowLiteSwift.podspec',
71]
72
Austin Anderson6afface2017-12-05 11:59:17 -080073
74class UserInputError(Exception):
75 pass
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070076
77
78def is_windows():
79 return platform.system() == 'Windows'
80
81
82def is_linux():
83 return platform.system() == 'Linux'
84
85
86def is_macos():
87 return platform.system() == 'Darwin'
88
89
90def is_ppc64le():
91 return platform.machine() == 'ppc64le'
92
93
Jonathan Hseu008910f2017-08-25 14:01:05 -070094def is_cygwin():
95 return platform.system().startswith('CYGWIN_NT')
96
97
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070098def get_input(question):
99 try:
100 try:
101 answer = raw_input(question)
102 except NameError:
103 answer = input(question) # pylint: disable=bad-builtin
104 except EOFError:
105 answer = ''
106 return answer
107
108
109def symlink_force(target, link_name):
110 """Force symlink, equivalent of 'ln -sf'.
111
112 Args:
113 target: items to link to.
114 link_name: name of the link.
115 """
116 try:
117 os.symlink(target, link_name)
118 except OSError as e:
119 if e.errno == errno.EEXIST:
120 os.remove(link_name)
121 os.symlink(target, link_name)
122 else:
123 raise e
124
125
126def sed_in_place(filename, old, new):
127 """Replace old string with new string in file.
128
129 Args:
130 filename: string for filename.
131 old: string to replace.
132 new: new string to replace to.
133 """
134 with open(filename, 'r') as f:
135 filedata = f.read()
136 newdata = filedata.replace(old, new)
137 with open(filename, 'w') as f:
138 f.write(newdata)
139
140
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700141def write_to_bazelrc(line):
142 with open(_TF_BAZELRC, 'a') as f:
143 f.write(line + '\n')
144
145
146def write_action_env_to_bazelrc(var_name, var):
147 write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var)))
148
149
Jonathan Hseu008910f2017-08-25 14:01:05 -0700150def run_shell(cmd, allow_non_zero=False):
151 if allow_non_zero:
152 try:
153 output = subprocess.check_output(cmd)
154 except subprocess.CalledProcessError as e:
155 output = e.output
156 else:
157 output = subprocess.check_output(cmd)
158 return output.decode('UTF-8').strip()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700159
160
161def cygpath(path):
162 """Convert path from posix to windows."""
Martin Wicked57572e2017-09-02 19:21:45 -0700163 return os.path.abspath(path).replace('\\', '/')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700164
165
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700166def get_python_path(environ_cp, python_bin_path):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700167 """Get the python site package paths."""
168 python_paths = []
169 if environ_cp.get('PYTHONPATH'):
170 python_paths = environ_cp.get('PYTHONPATH').split(':')
171 try:
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700172 library_paths = run_shell([
173 python_bin_path, '-c',
174 'import site; print("\\n".join(site.getsitepackages()))'
175 ]).split('\n')
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700176 except subprocess.CalledProcessError:
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700177 library_paths = [
178 run_shell([
179 python_bin_path, '-c',
180 'from distutils.sysconfig import get_python_lib;'
181 'print(get_python_lib())'
182 ])
183 ]
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700184
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700185 all_paths = set(python_paths + library_paths)
186
187 paths = []
188 for path in all_paths:
189 if os.path.isdir(path):
190 paths.append(path)
191 return paths
192
193
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700194def get_python_major_version(python_bin_path):
195 """Get the python major version."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700196 return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])'])
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700197
198
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -0700199def setup_python(environ_cp):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700200 """Setup python related env variables."""
201 # Get PYTHON_BIN_PATH, default is the current running python.
202 default_python_bin_path = sys.executable
203 ask_python_bin_path = ('Please specify the location of python. [Default is '
204 '%s]: ') % default_python_bin_path
205 while True:
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700206 python_bin_path = get_from_env_or_user_or_default(environ_cp,
207 'PYTHON_BIN_PATH',
208 ask_python_bin_path,
209 default_python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700210 # Check if the path is valid
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700211 if os.path.isfile(python_bin_path) and os.access(python_bin_path, os.X_OK):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700212 break
213 elif not os.path.exists(python_bin_path):
214 print('Invalid python path: %s cannot be found.' % python_bin_path)
215 else:
216 print('%s is not executable. Is it the python binary?' % python_bin_path)
217 environ_cp['PYTHON_BIN_PATH'] = ''
218
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700219 # Convert python path to Windows style before checking lib and version
Martin Wicked57572e2017-09-02 19:21:45 -0700220 if is_windows() or is_cygwin():
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700221 python_bin_path = cygpath(python_bin_path)
222
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700223 # Get PYTHON_LIB_PATH
224 python_lib_path = environ_cp.get('PYTHON_LIB_PATH')
225 if not python_lib_path:
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700226 python_lib_paths = get_python_path(environ_cp, python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700227 if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1':
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700228 python_lib_path = python_lib_paths[0]
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700229 else:
Jonathan Hseu008910f2017-08-25 14:01:05 -0700230 print('Found possible Python library paths:\n %s' %
231 '\n '.join(python_lib_paths))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700232 default_python_lib_path = python_lib_paths[0]
233 python_lib_path = get_input(
Jonathan Hseu008910f2017-08-25 14:01:05 -0700234 'Please input the desired Python library path to use. '
235 'Default is [%s]\n' % python_lib_paths[0])
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700236 if not python_lib_path:
237 python_lib_path = default_python_lib_path
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700238 environ_cp['PYTHON_LIB_PATH'] = python_lib_path
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700239
A. Unique TensorFlowercfccbdb2019-06-24 11:01:58 -0700240 python_major_version = get_python_major_version(python_bin_path)
241 if python_major_version == '2':
242 write_to_bazelrc('build --host_force_python=PY2')
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700243
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700244 # Convert python path to Windows style before writing into bazel.rc
Martin Wicked57572e2017-09-02 19:21:45 -0700245 if is_windows() or is_cygwin():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700246 python_lib_path = cygpath(python_lib_path)
247
248 # Set-up env variables used by python_configure.bzl
249 write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path)
250 write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path)
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -0700251 write_to_bazelrc('build --python_path=\"%s"' % python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700252 environ_cp['PYTHON_BIN_PATH'] = python_bin_path
253
William D. Ironsdcc76a52018-11-20 10:35:18 -0600254 # If choosen python_lib_path is from a path specified in the PYTHONPATH
255 # variable, need to tell bazel to include PYTHONPATH
256 if environ_cp.get('PYTHONPATH'):
257 python_paths = environ_cp.get('PYTHONPATH').split(':')
258 if python_lib_path in python_paths:
TensorFlower Gardener968cd182018-11-28 11:33:16 -0800259 write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH'))
William D. Ironsdcc76a52018-11-20 10:35:18 -0600260
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700261 # Write tools/python_bin_path.sh
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700262 with open(
263 os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'),
264 'w') as f:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700265 f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path)
266
267
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -0700268def reset_tf_configure_bazelrc():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700269 """Reset file that contains customized config settings."""
270 open(_TF_BAZELRC, 'w').close()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700271
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800272
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700273def cleanup_makefile():
274 """Delete any leftover BUILD files from the Makefile build.
275
276 These files could interfere with Bazel parsing.
277 """
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700278 makefile_download_dir = os.path.join(_TF_WORKSPACE_ROOT, 'tensorflow',
279 'contrib', 'makefile', 'downloads')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700280 if os.path.isdir(makefile_download_dir):
281 for root, _, filenames in os.walk(makefile_download_dir):
282 for f in filenames:
283 if f.endswith('BUILD'):
284 os.remove(os.path.join(root, f))
285
286
287def get_var(environ_cp,
288 var_name,
289 query_item,
290 enabled_by_default,
291 question=None,
292 yes_reply=None,
293 no_reply=None):
294 """Get boolean input from user.
295
296 If var_name is not set in env, ask user to enable query_item or not. If the
297 response is empty, use the default.
298
299 Args:
300 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530301 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
302 query_item: string for feature related to the variable, e.g. "CUDA for
303 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700304 enabled_by_default: boolean for default behavior.
305 question: optional string for how to ask for user input.
Michael Cased90054e2018-02-07 14:36:00 -0800306 yes_reply: optional string for reply when feature is enabled.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700307 no_reply: optional string for reply when feature is disabled.
308
309 Returns:
310 boolean value of the variable.
Frank Chenc4ef9272018-01-10 11:36:52 -0800311
312 Raises:
313 UserInputError: if an environment variable is set, but it cannot be
314 interpreted as a boolean indicator, assume that the user has made a
315 scripting error, and will continue to provide invalid input.
316 Raise the error to avoid infinitely looping.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700317 """
318 if not question:
319 question = 'Do you wish to build TensorFlow with %s support?' % query_item
320 if not yes_reply:
321 yes_reply = '%s support will be enabled for TensorFlow.' % query_item
322 if not no_reply:
323 no_reply = 'No %s' % yes_reply
324
325 yes_reply += '\n'
326 no_reply += '\n'
327
328 if enabled_by_default:
329 question += ' [Y/n]: '
330 else:
331 question += ' [y/N]: '
332
333 var = environ_cp.get(var_name)
Frank Chenc4ef9272018-01-10 11:36:52 -0800334 if var is not None:
335 var_content = var.strip().lower()
336 true_strings = ('1', 't', 'true', 'y', 'yes')
337 false_strings = ('0', 'f', 'false', 'n', 'no')
338 if var_content in true_strings:
339 var = True
340 elif var_content in false_strings:
341 var = False
342 else:
343 raise UserInputError(
344 'Environment variable %s must be set as a boolean indicator.\n'
345 'The following are accepted as TRUE : %s.\n'
346 'The following are accepted as FALSE: %s.\n'
A. Unique TensorFlowered297342019-03-15 11:25:28 -0700347 'Current value is %s.' %
348 (var_name, ', '.join(true_strings), ', '.join(false_strings), var))
Frank Chenc4ef9272018-01-10 11:36:52 -0800349
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700350 while var is None:
351 user_input_origin = get_input(question)
352 user_input = user_input_origin.strip().lower()
353 if user_input == 'y':
354 print(yes_reply)
355 var = True
356 elif user_input == 'n':
357 print(no_reply)
358 var = False
359 elif not user_input:
360 if enabled_by_default:
361 print(yes_reply)
362 var = True
363 else:
364 print(no_reply)
365 var = False
366 else:
367 print('Invalid selection: %s' % user_input_origin)
368 return var
369
370
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700371def set_build_var(environ_cp,
372 var_name,
373 query_item,
374 option_name,
375 enabled_by_default,
376 bazel_config_name=None):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700377 """Set if query_item will be enabled for the build.
378
379 Ask user if query_item will be enabled. Default is used if no input is given.
380 Set subprocess environment variable and write to .bazelrc if enabled.
381
382 Args:
383 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530384 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
385 query_item: string for feature related to the variable, e.g. "CUDA for
386 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700387 option_name: string for option to define in .bazelrc.
388 enabled_by_default: boolean for default behavior.
Michael Case98850a52017-09-14 13:35:57 -0700389 bazel_config_name: Name for Bazel --config argument to enable build feature.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700390 """
391
392 var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default)))
393 environ_cp[var_name] = var
394 if var == '1':
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700395 write_to_bazelrc('build:%s --define %s=true' %
396 (bazel_config_name, option_name))
Yifei Fengec451f52018-10-05 12:53:50 -0700397 write_to_bazelrc('build --config=%s' % bazel_config_name)
Michael Case98850a52017-09-14 13:35:57 -0700398 elif bazel_config_name is not None:
399 # TODO(mikecase): Migrate all users of configure.py to use --config Bazel
400 # options and not to set build configs through environment variables.
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700401 write_to_bazelrc('build:%s --define %s=true' %
402 (bazel_config_name, option_name))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700403
404
405def set_action_env_var(environ_cp,
406 var_name,
407 query_item,
408 enabled_by_default,
409 question=None,
410 yes_reply=None,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700411 no_reply=None,
412 bazel_config_name=None):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700413 """Set boolean action_env variable.
414
415 Ask user if query_item will be enabled. Default is used if no input is given.
416 Set environment variable and write to .bazelrc.
417
418 Args:
419 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530420 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
421 query_item: string for feature related to the variable, e.g. "CUDA for
422 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700423 enabled_by_default: boolean for default behavior.
424 question: optional string for how to ask for user input.
Michael Cased90054e2018-02-07 14:36:00 -0800425 yes_reply: optional string for reply when feature is enabled.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700426 no_reply: optional string for reply when feature is disabled.
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700427 bazel_config_name: adding config to .bazelrc instead of action_env.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700428 """
429 var = int(
430 get_var(environ_cp, var_name, query_item, enabled_by_default, question,
431 yes_reply, no_reply))
432
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700433 if not bazel_config_name:
434 write_action_env_to_bazelrc(var_name, var)
435 elif var:
436 write_to_bazelrc('build --config=%s' % bazel_config_name)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700437 environ_cp[var_name] = str(var)
438
439
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700440def convert_version_to_int(version):
441 """Convert a version number to a integer that can be used to compare.
442
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700443 Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The
444 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored.
445
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700446 Args:
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700447 version: a version to be converted
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700448
449 Returns:
450 An integer if converted successfully, otherwise return None.
451 """
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700452 version = version.split('-')[0]
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700453 version_segments = version.split('.')
Austin Anderson87ea41d2019-04-04 10:03:50 -0700454 # Treat "0.24" as "0.24.0"
455 if len(version_segments) == 2:
456 version_segments.append('0')
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700457 for seg in version_segments:
458 if not seg.isdigit():
459 return None
460
461 version_str = ''.join(['%03d' % int(seg) for seg in version_segments])
462 return int(version_str)
463
464
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800465def check_bazel_version(min_version, max_version):
466 """Check installed bazel version is between min_version and max_version.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700467
468 Args:
Mihai Maruseac4db860f2019-04-19 08:52:10 -0700469 min_version: string for minimum bazel version (must exist!).
470 max_version: string for maximum bazel version (must exist!).
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700471
472 Returns:
473 The bazel version detected.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700474 """
Jonathan Hseu008910f2017-08-25 14:01:05 -0700475 if which('bazel') is None:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700476 print('Cannot find bazel. Please install bazel.')
477 sys.exit(0)
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700478 curr_version = run_shell(
479 ['bazel', '--batch', '--bazelrc=/dev/null', 'version'])
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700480
481 for line in curr_version.split('\n'):
482 if 'Build label: ' in line:
483 curr_version = line.split('Build label: ')[1]
484 break
485
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700486 min_version_int = convert_version_to_int(min_version)
487 curr_version_int = convert_version_to_int(curr_version)
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800488 max_version_int = convert_version_to_int(max_version)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700489
490 # Check if current bazel version can be detected properly.
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700491 if not curr_version_int:
492 print('WARNING: current bazel installation is not a release version.')
493 print('Make sure you are running at least bazel %s' % min_version)
494 return curr_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700495
Michael Cased94271a2017-08-22 17:26:52 -0700496 print('You have bazel %s installed.' % curr_version)
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700497
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700498 if curr_version_int < min_version_int:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700499 print('Please upgrade your bazel installation to version %s or higher to '
500 'build TensorFlow!' % min_version)
Jason Zamanb41761c2018-10-14 11:28:53 +0800501 sys.exit(1)
TensorFlower Gardener78c246b2018-12-13 12:37:42 -0800502 if (curr_version_int > max_version_int and
503 'TF_IGNORE_MAX_BAZEL_VERSION' not in os.environ):
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800504 print('Please downgrade your bazel installation to version %s or lower to '
Mihai Maruseace0963c42018-12-20 14:27:40 -0800505 'build TensorFlow! To downgrade: download the installer for the old '
506 'version (from https://github.com/bazelbuild/bazel/releases) then '
507 'run the installer.' % max_version)
Jason Zamanb41761c2018-10-14 11:28:53 +0800508 sys.exit(1)
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700509 return curr_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700510
511
512def set_cc_opt_flags(environ_cp):
513 """Set up architecture-dependent optimization flags.
514
515 Also append CC optimization flags to bazel.rc..
516
517 Args:
518 environ_cp: copy of the os.environ.
519 """
520 if is_ppc64le():
521 # gcc on ppc64le does not support -march, use mcpu instead
522 default_cc_opt_flags = '-mcpu=native'
A. Unique TensorFlower1bba94a2018-04-04 15:45:20 -0700523 elif is_windows():
524 default_cc_opt_flags = '/arch:AVX'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700525 else:
Justin Lebar9ef04f52018-10-10 18:52:45 -0700526 default_cc_opt_flags = '-march=native -Wno-sign-compare'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700527 question = ('Please specify optimization flags to use during compilation when'
528 ' bazel option "--config=opt" is specified [Default is %s]: '
529 ) % default_cc_opt_flags
530 cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS',
531 question, default_cc_opt_flags)
532 for opt in cc_opt_flags.split():
Michael Case00177422017-11-10 13:14:03 -0800533 write_to_bazelrc('build:opt --copt=%s' % opt)
534 # It should be safe on the same build host.
A. Unique TensorFlower1bba94a2018-04-04 15:45:20 -0700535 if not is_ppc64le() and not is_windows():
Shanqing Cai71445712018-03-12 19:33:52 -0700536 write_to_bazelrc('build:opt --host_copt=-march=native')
Michael Casebb3355d2017-11-09 08:46:31 -0800537 write_to_bazelrc('build:opt --define with_default_optimizations=true')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700538
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700539
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700540def set_tf_cuda_clang(environ_cp):
541 """set TF_CUDA_CLANG action_env.
542
543 Args:
544 environ_cp: copy of the os.environ.
545 """
546 question = 'Do you want to use clang as CUDA compiler?'
547 yes_reply = 'Clang will be used as CUDA compiler.'
548 no_reply = 'nvcc will be used as CUDA compiler.'
549 set_action_env_var(
550 environ_cp,
551 'TF_CUDA_CLANG',
552 None,
553 False,
554 question=question,
555 yes_reply=yes_reply,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700556 no_reply=no_reply,
557 bazel_config_name='cuda_clang')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700558
559
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800560def set_tf_download_clang(environ_cp):
561 """Set TF_DOWNLOAD_CLANG action_env."""
Ilya Biryukov9e651e42018-03-22 05:33:42 -0700562 question = 'Do you wish to download a fresh release of clang? (Experimental)'
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800563 yes_reply = 'Clang will be downloaded and used to compile tensorflow.'
564 no_reply = 'Clang will not be downloaded.'
565 set_action_env_var(
566 environ_cp,
567 'TF_DOWNLOAD_CLANG',
568 None,
569 False,
570 question=question,
571 yes_reply=yes_reply,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700572 no_reply=no_reply,
573 bazel_config_name='download_clang')
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800574
575
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700576def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var,
577 var_default):
578 """Get var_name either from env, or user or default.
579
580 If var_name has been set as environment variable, use the preset value, else
581 ask for user input. If no input is provided, the default is used.
582
583 Args:
584 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530585 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700586 ask_for_var: string for how to ask for user input.
587 var_default: default value string.
588
589 Returns:
590 string value for var_name
591 """
592 var = environ_cp.get(var_name)
593 if not var:
594 var = get_input(ask_for_var)
Michael Cased94271a2017-08-22 17:26:52 -0700595 print('\n')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700596 if not var:
597 var = var_default
598 return var
599
600
601def set_clang_cuda_compiler_path(environ_cp):
602 """Set CLANG_CUDA_COMPILER_PATH."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700603 default_clang_path = which('clang') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700604 ask_clang_path = ('Please specify which clang should be used as device and '
605 'host compiler. [Default is %s]: ') % default_clang_path
606
607 while True:
608 clang_cuda_compiler_path = get_from_env_or_user_or_default(
609 environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path,
610 default_clang_path)
611 if os.path.exists(clang_cuda_compiler_path):
612 break
613
614 # Reset and retry
615 print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path)
616 environ_cp['CLANG_CUDA_COMPILER_PATH'] = ''
617
618 # Set CLANG_CUDA_COMPILER_PATH
619 environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path
620 write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH',
621 clang_cuda_compiler_path)
622
623
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700624def prompt_loop_or_load_from_env(environ_cp,
625 var_name,
626 var_default,
627 ask_for_var,
628 check_success,
629 error_msg,
630 suppress_default_error=False,
Austin Anderson58b236b2019-10-22 10:36:10 -0700631 resolve_symlinks=False,
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700632 n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS):
Austin Anderson6afface2017-12-05 11:59:17 -0800633 """Loop over user prompts for an ENV param until receiving a valid response.
634
635 For the env param var_name, read from the environment or verify user input
636 until receiving valid input. When done, set var_name in the environ_cp to its
637 new value.
638
639 Args:
640 environ_cp: (Dict) copy of the os.environ.
641 var_name: (String) string for name of environment variable, e.g. "TF_MYVAR".
642 var_default: (String) default value string.
643 ask_for_var: (String) string for how to ask for user input.
644 check_success: (Function) function that takes one argument and returns a
645 boolean. Should return True if the value provided is considered valid. May
646 contain a complex error message if error_msg does not provide enough
647 information. In that case, set suppress_default_error to True.
648 error_msg: (String) String with one and only one '%s'. Formatted with each
649 invalid response upon check_success(input) failure.
650 suppress_default_error: (Bool) Suppress the above error message in favor of
651 one from the check_success function.
Austin Anderson58b236b2019-10-22 10:36:10 -0700652 resolve_symlinks: (Bool) Translate symbolic links into the real filepath.
Austin Anderson6afface2017-12-05 11:59:17 -0800653 n_ask_attempts: (Integer) Number of times to query for valid input before
654 raising an error and quitting.
655
656 Returns:
657 [String] The value of var_name after querying for input.
658
659 Raises:
660 UserInputError: if a query has been attempted n_ask_attempts times without
Frank Chenc4ef9272018-01-10 11:36:52 -0800661 success, assume that the user has made a scripting error, and will
662 continue to provide invalid input. Raise the error to avoid infinitely
663 looping.
Austin Anderson6afface2017-12-05 11:59:17 -0800664 """
665 default = environ_cp.get(var_name) or var_default
666 full_query = '%s [Default is %s]: ' % (
667 ask_for_var,
668 default,
669 )
670
671 for _ in range(n_ask_attempts):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700672 val = get_from_env_or_user_or_default(environ_cp, var_name, full_query,
Austin Anderson6afface2017-12-05 11:59:17 -0800673 default)
674 if check_success(val):
675 break
676 if not suppress_default_error:
677 print(error_msg % val)
678 environ_cp[var_name] = ''
679 else:
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700680 raise UserInputError('Invalid %s setting was provided %d times in a row. '
681 'Assuming to be a scripting mistake.' %
682 (var_name, n_ask_attempts))
Austin Anderson6afface2017-12-05 11:59:17 -0800683
Austin Anderson58b236b2019-10-22 10:36:10 -0700684 if resolve_symlinks and os.path.islink(val):
685 val = os.path.realpath(val)
Austin Anderson6afface2017-12-05 11:59:17 -0800686 environ_cp[var_name] = val
687 return val
688
689
690def create_android_ndk_rule(environ_cp):
691 """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule."""
692 if is_windows() or is_cygwin():
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700693 default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' %
694 environ_cp['APPDATA'])
Austin Anderson6afface2017-12-05 11:59:17 -0800695 elif is_macos():
696 default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME']
697 else:
698 default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME']
699
700 def valid_ndk_path(path):
701 return (os.path.exists(path) and
702 os.path.exists(os.path.join(path, 'source.properties')))
703
704 android_ndk_home_path = prompt_loop_or_load_from_env(
705 environ_cp,
706 var_name='ANDROID_NDK_HOME',
707 var_default=default_ndk_path,
708 ask_for_var='Please specify the home path of the Android NDK to use.',
709 check_success=valid_ndk_path,
710 error_msg=('The path %s or its child file "source.properties" '
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700711 'does not exist.'))
Michael Case51053502018-06-05 17:47:19 -0700712 write_action_env_to_bazelrc('ANDROID_NDK_HOME', android_ndk_home_path)
Jared Dukea0104b72019-04-04 12:23:58 -0700713 write_action_env_to_bazelrc(
714 'ANDROID_NDK_API_LEVEL',
715 get_ndk_api_level(environ_cp, android_ndk_home_path))
Austin Anderson6afface2017-12-05 11:59:17 -0800716
717
718def create_android_sdk_rule(environ_cp):
719 """Set Android variables and write Android SDK WORKSPACE rule."""
720 if is_windows() or is_cygwin():
721 default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA'])
722 elif is_macos():
Shashi Shekharc0ff0cc2018-07-17 09:00:24 -0700723 default_sdk_path = '%s/library/Android/Sdk' % environ_cp['HOME']
Austin Anderson6afface2017-12-05 11:59:17 -0800724 else:
725 default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME']
726
727 def valid_sdk_path(path):
728 return (os.path.exists(path) and
729 os.path.exists(os.path.join(path, 'platforms')) and
730 os.path.exists(os.path.join(path, 'build-tools')))
731
732 android_sdk_home_path = prompt_loop_or_load_from_env(
733 environ_cp,
734 var_name='ANDROID_SDK_HOME',
735 var_default=default_sdk_path,
736 ask_for_var='Please specify the home path of the Android SDK to use.',
737 check_success=valid_sdk_path,
738 error_msg=('Either %s does not exist, or it does not contain the '
739 'subdirectories "platforms" and "build-tools".'))
740
741 platforms = os.path.join(android_sdk_home_path, 'platforms')
742 api_levels = sorted(os.listdir(platforms))
743 api_levels = [x.replace('android-', '') for x in api_levels]
744
745 def valid_api_level(api_level):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700746 return os.path.exists(
747 os.path.join(android_sdk_home_path, 'platforms',
748 'android-' + api_level))
Austin Anderson6afface2017-12-05 11:59:17 -0800749
750 android_api_level = prompt_loop_or_load_from_env(
751 environ_cp,
752 var_name='ANDROID_API_LEVEL',
753 var_default=api_levels[-1],
754 ask_for_var=('Please specify the Android SDK API level to use. '
755 '[Available levels: %s]') % api_levels,
756 check_success=valid_api_level,
757 error_msg='Android-%s is not present in the SDK path.')
758
759 build_tools = os.path.join(android_sdk_home_path, 'build-tools')
760 versions = sorted(os.listdir(build_tools))
761
762 def valid_build_tools(version):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700763 return os.path.exists(
764 os.path.join(android_sdk_home_path, 'build-tools', version))
Austin Anderson6afface2017-12-05 11:59:17 -0800765
766 android_build_tools_version = prompt_loop_or_load_from_env(
767 environ_cp,
768 var_name='ANDROID_BUILD_TOOLS_VERSION',
769 var_default=versions[-1],
770 ask_for_var=('Please specify an Android build tools version to use. '
771 '[Available versions: %s]') % versions,
772 check_success=valid_build_tools,
773 error_msg=('The selected SDK does not have build-tools version %s '
774 'available.'))
775
Michael Case51053502018-06-05 17:47:19 -0700776 write_action_env_to_bazelrc('ANDROID_BUILD_TOOLS_VERSION',
777 android_build_tools_version)
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700778 write_action_env_to_bazelrc('ANDROID_SDK_API_LEVEL', android_api_level)
779 write_action_env_to_bazelrc('ANDROID_SDK_HOME', android_sdk_home_path)
Austin Anderson6afface2017-12-05 11:59:17 -0800780
781
Jared Dukea0104b72019-04-04 12:23:58 -0700782def get_ndk_api_level(environ_cp, android_ndk_home_path):
783 """Gets the appropriate NDK API level to use for the provided Android NDK path."""
784
785 # First check to see if we're using a blessed version of the NDK.
Austin Anderson6afface2017-12-05 11:59:17 -0800786 properties_path = '%s/source.properties' % android_ndk_home_path
787 if is_windows() or is_cygwin():
788 properties_path = cygpath(properties_path)
789 with open(properties_path, 'r') as f:
790 filedata = f.read()
791
792 revision = re.search(r'Pkg.Revision = (\d+)', filedata)
793 if revision:
Jared Dukea0104b72019-04-04 12:23:58 -0700794 ndk_version = revision.group(1)
Michael Case51053502018-06-05 17:47:19 -0700795 else:
796 raise Exception('Unable to parse NDK revision.')
Jared Dukea0104b72019-04-04 12:23:58 -0700797 if int(ndk_version) not in _SUPPORTED_ANDROID_NDK_VERSIONS:
798 print('WARNING: The NDK version in %s is %s, which is not '
799 'supported by Bazel (officially supported versions: %s). Please use '
800 'another version. Compiling Android targets may result in confusing '
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700801 'errors.\n' %
802 (android_ndk_home_path, ndk_version, _SUPPORTED_ANDROID_NDK_VERSIONS))
Jared Dukea0104b72019-04-04 12:23:58 -0700803
804 # Now grab the NDK API level to use. Note that this is different from the
805 # SDK API level, as the NDK API level is effectively the *min* target SDK
806 # version.
807 platforms = os.path.join(android_ndk_home_path, 'platforms')
808 api_levels = sorted(os.listdir(platforms))
809 api_levels = [
810 x.replace('android-', '') for x in api_levels if 'android-' in x
811 ]
812
813 def valid_api_level(api_level):
814 return os.path.exists(
815 os.path.join(android_ndk_home_path, 'platforms',
816 'android-' + api_level))
817
818 android_ndk_api_level = prompt_loop_or_load_from_env(
819 environ_cp,
820 var_name='ANDROID_NDK_API_LEVEL',
Jared Dukea1309f42019-11-15 17:09:35 -0800821 var_default='21', # 21 is required for ARM64 support.
Jared Dukea0104b72019-04-04 12:23:58 -0700822 ask_for_var=('Please specify the (min) Android NDK API level to use. '
823 '[Available levels: %s]') % api_levels,
824 check_success=valid_api_level,
825 error_msg='Android-%s is not present in the NDK path.')
826
827 return android_ndk_api_level
Austin Anderson6afface2017-12-05 11:59:17 -0800828
829
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700830def set_gcc_host_compiler_path(environ_cp):
831 """Set GCC_HOST_COMPILER_PATH."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700832 default_gcc_host_compiler_path = which('gcc') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700833 cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH')
834
835 if os.path.islink(cuda_bin_symlink):
836 # os.readlink is only available in linux
Jonathan Hseu008910f2017-08-25 14:01:05 -0700837 default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700838
Austin Anderson6afface2017-12-05 11:59:17 -0800839 gcc_host_compiler_path = prompt_loop_or_load_from_env(
840 environ_cp,
841 var_name='GCC_HOST_COMPILER_PATH',
842 var_default=default_gcc_host_compiler_path,
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800843 ask_for_var='Please specify which gcc should be used by nvcc as the host compiler.',
Austin Anderson6afface2017-12-05 11:59:17 -0800844 check_success=os.path.exists,
Austin Anderson58b236b2019-10-22 10:36:10 -0700845 resolve_symlinks=True,
Austin Anderson6afface2017-12-05 11:59:17 -0800846 error_msg='Invalid gcc path. %s cannot be found.',
847 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700848
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700849 write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path)
850
851
Ankur Taly0e6f39d2018-02-16 18:22:55 -0800852def reformat_version_sequence(version_str, sequence_count):
853 """Reformat the version string to have the given number of sequences.
854
855 For example:
856 Given (7, 2) -> 7.0
857 (7.0.1, 2) -> 7.0
858 (5, 1) -> 5
859 (5.0.3.2, 1) -> 5
860
861 Args:
862 version_str: String, the version string.
863 sequence_count: int, an integer.
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700864
Ankur Taly0e6f39d2018-02-16 18:22:55 -0800865 Returns:
866 string, reformatted version string.
867 """
868 v = version_str.split('.')
869 if len(v) < sequence_count:
870 v = v + (['0'] * (sequence_count - len(v)))
871
872 return '.'.join(v[:sequence_count])
873
874
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700875def set_tf_cuda_paths(environ_cp):
876 """Set TF_CUDA_PATHS."""
877 ask_cuda_paths = (
878 'Please specify the comma-separated list of base paths to look for CUDA '
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700879 'libraries and headers. [Leave empty to use the default]: ')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700880 tf_cuda_paths = get_from_env_or_user_or_default(environ_cp, 'TF_CUDA_PATHS',
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700881 ask_cuda_paths, '')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700882 if tf_cuda_paths:
883 environ_cp['TF_CUDA_PATHS'] = tf_cuda_paths
884
885
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700886def set_tf_cuda_version(environ_cp):
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700887 """Set TF_CUDA_VERSION."""
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700888 ask_cuda_version = (
889 'Please specify the CUDA SDK version you want to use. '
890 '[Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700891 tf_cuda_version = get_from_env_or_user_or_default(environ_cp,
892 'TF_CUDA_VERSION',
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700893 ask_cuda_version,
894 _DEFAULT_CUDA_VERSION)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700895 environ_cp['TF_CUDA_VERSION'] = tf_cuda_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700896
897
Yifei Fengb1d8c592017-11-22 13:42:21 -0800898def set_tf_cudnn_version(environ_cp):
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700899 """Set TF_CUDNN_VERSION."""
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700900 ask_cudnn_version = (
901 'Please specify the cuDNN version you want to use. '
902 '[Leave empty to default to cuDNN %s]: ') % _DEFAULT_CUDNN_VERSION
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700903 tf_cudnn_version = get_from_env_or_user_or_default(environ_cp,
904 'TF_CUDNN_VERSION',
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700905 ask_cudnn_version,
906 _DEFAULT_CUDNN_VERSION)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700907 environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700908
909
Mingxing Tan1e7b0e42018-06-28 19:13:20 -0700910def is_cuda_compatible(lib, cuda_ver, cudnn_ver):
911 """Check compatibility between given library and cudnn/cudart libraries."""
912 ldd_bin = which('ldd') or '/usr/bin/ldd'
913 ldd_out = run_shell([ldd_bin, lib], True)
914 ldd_out = ldd_out.split(os.linesep)
915 cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$')
916 cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$')
917 cudnn = None
918 cudart = None
919 cudnn_ok = True # assume no cudnn dependency by default
920 cuda_ok = True # assume no cuda dependency by default
921 for line in ldd_out:
922 if 'libcudnn.so' in line:
923 cudnn = cudnn_pattern.search(line)
924 cudnn_ok = False
925 elif 'libcudart.so' in line:
926 cudart = cuda_pattern.search(line)
927 cuda_ok = False
928 if cudnn and len(cudnn.group(1)):
929 cudnn = convert_version_to_int(cudnn.group(1))
930 if cudart and len(cudart.group(1)):
931 cudart = convert_version_to_int(cudart.group(1))
932 if cudnn is not None:
933 cudnn_ok = (cudnn == cudnn_ver)
934 if cudart is not None:
935 cuda_ok = (cudart == cuda_ver)
936 return cudnn_ok and cuda_ok
937
938
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700939def set_tf_tensorrt_version(environ_cp):
940 """Set TF_TENSORRT_VERSION."""
Guangda Lai76f69382018-01-25 23:59:19 -0800941 if not is_linux():
942 raise ValueError('Currently TensorRT is only supported on Linux platform.')
943
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700944 if not int(environ_cp.get('TF_NEED_TENSORRT', False)):
Guangda Lai76f69382018-01-25 23:59:19 -0800945 return
946
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700947 ask_tensorrt_version = (
948 'Please specify the TensorRT version you want to use. '
Guangda Laifaa93ac2019-10-23 15:07:12 -0700949 '[Leave empty to default to TensorRT %s]: ') % _DEFAULT_TENSORRT_VERSION
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700950 tf_tensorrt_version = get_from_env_or_user_or_default(
951 environ_cp, 'TF_TENSORRT_VERSION', ask_tensorrt_version,
952 _DEFAULT_TENSORRT_VERSION)
Guangda Lai76f69382018-01-25 23:59:19 -0800953 environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version
Guangda Lai76f69382018-01-25 23:59:19 -0800954
955
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700956def set_tf_nccl_version(environ_cp):
957 """Set TF_NCCL_VERSION."""
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700958 if not is_linux():
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700959 raise ValueError('Currently NCCL is only supported on Linux platform.')
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700960
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700961 if 'TF_NCCL_VERSION' in environ_cp:
962 return
963
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700964 ask_nccl_version = (
A. Unique TensorFlower53faa312018-10-05 08:46:54 -0700965 'Please specify the locally installed NCCL version you want to use. '
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700966 '[Leave empty to use http://github.com/nvidia/nccl]: ')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700967 tf_nccl_version = get_from_env_or_user_or_default(environ_cp,
968 'TF_NCCL_VERSION',
969 ask_nccl_version, '')
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700970 environ_cp['TF_NCCL_VERSION'] = tf_nccl_version
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800971
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700972
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700973def get_native_cuda_compute_capabilities(environ_cp):
974 """Get native cuda compute capabilities.
975
976 Args:
977 environ_cp: copy of the os.environ.
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700978
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700979 Returns:
980 string of native cuda compute capabilities, separated by comma.
981 """
982 device_query_bin = os.path.join(
983 environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery')
Jonathan Hseu008910f2017-08-25 14:01:05 -0700984 if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK):
985 try:
986 output = run_shell(device_query_bin).split('\n')
987 pattern = re.compile('[0-9]*\\.[0-9]*')
988 output = [pattern.search(x) for x in output if 'Capability' in x]
989 output = ','.join(x.group() for x in output if x is not None)
990 except subprocess.CalledProcessError:
991 output = ''
992 else:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700993 output = ''
994 return output
995
996
997def set_tf_cuda_compute_capabilities(environ_cp):
998 """Set TF_CUDA_COMPUTE_CAPABILITIES."""
999 while True:
1000 native_cuda_compute_capabilities = get_native_cuda_compute_capabilities(
1001 environ_cp)
1002 if not native_cuda_compute_capabilities:
1003 default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES
1004 else:
1005 default_cuda_compute_capabilities = native_cuda_compute_capabilities
1006
1007 ask_cuda_compute_capabilities = (
1008 'Please specify a list of comma-separated '
P Sudeepam52093562019-02-17 17:34:01 +05301009 'CUDA compute capabilities you want to '
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001010 'build with.\nYou can find the compute '
1011 'capability of your device at: '
1012 'https://developer.nvidia.com/cuda-gpus.\nPlease'
1013 ' note that each additional compute '
1014 'capability significantly increases your '
P Sudeepam52093562019-02-17 17:34:01 +05301015 'build time and binary size, and that '
1016 'TensorFlow only supports compute '
P Sudeepam765ceda2019-02-17 17:39:08 +05301017 'capabilities >= 3.5 [Default is: %s]: ' %
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001018 default_cuda_compute_capabilities)
1019 tf_cuda_compute_capabilities = get_from_env_or_user_or_default(
1020 environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES',
1021 ask_cuda_compute_capabilities, default_cuda_compute_capabilities)
1022 # Check whether all capabilities from the input is valid
1023 all_valid = True
Maciejd0f5bc12018-04-30 22:30:58 -05001024 # Remove all whitespace characters before splitting the string
Michael Case51053502018-06-05 17:47:19 -07001025 # that users may insert by accident, as this will result in error
Maciejd0f5bc12018-04-30 22:30:58 -05001026 tf_cuda_compute_capabilities = ''.join(tf_cuda_compute_capabilities.split())
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001027 for compute_capability in tf_cuda_compute_capabilities.split(','):
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001028 m = re.match('[0-9]+.[0-9]+', compute_capability)
1029 if not m:
Austin Anderson32202dc2019-02-19 10:46:27 -08001030 print('Invalid compute capability: %s' % compute_capability)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001031 all_valid = False
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001032 else:
P Sudeepam52093562019-02-17 17:34:01 +05301033 ver = float(m.group(0))
A. Unique TensorFlower8dc2d0e2019-03-12 01:41:05 -07001034 if ver < 3.0:
1035 print('ERROR: TensorFlow only supports CUDA compute capabilities 3.0 '
Austin Anderson32202dc2019-02-19 10:46:27 -08001036 'and higher. Please re-specify the list of compute '
1037 'capabilities excluding version %s.' % ver)
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001038 all_valid = False
A. Unique TensorFlower8dc2d0e2019-03-12 01:41:05 -07001039 if ver < 3.5:
1040 print('WARNING: XLA does not support CUDA compute capabilities '
1041 'lower than 3.5. Disable XLA when running on older GPUs.')
P Sudeepam765ceda2019-02-17 17:39:08 +05301042
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001043 if all_valid:
1044 break
1045
1046 # Reset and Retry
1047 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = ''
1048
1049 # Set TF_CUDA_COMPUTE_CAPABILITIES
1050 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities
1051 write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES',
1052 tf_cuda_compute_capabilities)
1053
1054
1055def set_other_cuda_vars(environ_cp):
1056 """Set other CUDA related variables."""
A. Unique TensorFlowerab391982018-07-11 04:52:49 -07001057 # If CUDA is enabled, always use GPU during build and test.
1058 if environ_cp.get('TF_CUDA_CLANG') == '1':
1059 write_to_bazelrc('build --config=cuda_clang')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001060 else:
A. Unique TensorFlowerab391982018-07-11 04:52:49 -07001061 write_to_bazelrc('build --config=cuda')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001062
1063
1064def set_host_cxx_compiler(environ_cp):
1065 """Set HOST_CXX_COMPILER."""
Jonathan Hseu008910f2017-08-25 14:01:05 -07001066 default_cxx_host_compiler = which('g++') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001067
Austin Anderson6afface2017-12-05 11:59:17 -08001068 host_cxx_compiler = prompt_loop_or_load_from_env(
1069 environ_cp,
1070 var_name='HOST_CXX_COMPILER',
1071 var_default=default_cxx_host_compiler,
1072 ask_for_var=('Please specify which C++ compiler should be used as the '
1073 'host C++ compiler.'),
1074 check_success=os.path.exists,
1075 error_msg='Invalid C++ compiler path. %s cannot be found.',
1076 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001077
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001078 write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler)
1079
1080
1081def set_host_c_compiler(environ_cp):
1082 """Set HOST_C_COMPILER."""
Jonathan Hseu008910f2017-08-25 14:01:05 -07001083 default_c_host_compiler = which('gcc') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001084
Austin Anderson6afface2017-12-05 11:59:17 -08001085 host_c_compiler = prompt_loop_or_load_from_env(
1086 environ_cp,
1087 var_name='HOST_C_COMPILER',
1088 var_default=default_c_host_compiler,
Shanqing Cai71445712018-03-12 19:33:52 -07001089 ask_for_var=('Please specify which C compiler should be used as the host '
Austin Anderson6afface2017-12-05 11:59:17 -08001090 'C compiler.'),
1091 check_success=os.path.exists,
1092 error_msg='Invalid C compiler path. %s cannot be found.',
1093 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001094
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001095 write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler)
1096
1097
1098def set_computecpp_toolkit_path(environ_cp):
1099 """Set COMPUTECPP_TOOLKIT_PATH."""
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001100
Austin Anderson6afface2017-12-05 11:59:17 -08001101 def toolkit_exists(toolkit_path):
1102 """Check if a computecpp toolkit path is valid."""
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001103 if is_linux():
1104 sycl_rt_lib_path = 'lib/libComputeCpp.so'
1105 else:
1106 sycl_rt_lib_path = ''
1107
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001108 sycl_rt_lib_path_full = os.path.join(toolkit_path, sycl_rt_lib_path)
Austin Anderson6afface2017-12-05 11:59:17 -08001109 exists = os.path.exists(sycl_rt_lib_path_full)
1110 if not exists:
1111 print('Invalid SYCL %s library path. %s cannot be found' %
1112 (_TF_OPENCL_VERSION, sycl_rt_lib_path_full))
1113 return exists
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001114
Austin Anderson6afface2017-12-05 11:59:17 -08001115 computecpp_toolkit_path = prompt_loop_or_load_from_env(
1116 environ_cp,
1117 var_name='COMPUTECPP_TOOLKIT_PATH',
1118 var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH,
1119 ask_for_var=(
1120 'Please specify the location where ComputeCpp for SYCL %s is '
1121 'installed.' % _TF_OPENCL_VERSION),
1122 check_success=toolkit_exists,
1123 error_msg='Invalid SYCL compiler path. %s cannot be found.',
1124 suppress_default_error=True)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001125
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001126 write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH',
1127 computecpp_toolkit_path)
1128
Michael Cased31531a2018-01-05 14:09:41 -08001129
Dandelion Man?90e42f32017-12-15 18:15:07 -08001130def set_trisycl_include_dir(environ_cp):
Michael Cased31531a2018-01-05 14:09:41 -08001131 """Set TRISYCL_INCLUDE_DIR."""
Frank Chenc4ef9272018-01-10 11:36:52 -08001132
Dandelion Man?90e42f32017-12-15 18:15:07 -08001133 ask_trisycl_include_dir = ('Please specify the location of the triSYCL '
1134 'include directory. (Use --config=sycl_trisycl '
1135 'when building with Bazel) '
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001136 '[Default is %s]: ') % (
1137 _DEFAULT_TRISYCL_INCLUDE_DIR)
Frank Chenc4ef9272018-01-10 11:36:52 -08001138
Dandelion Man?90e42f32017-12-15 18:15:07 -08001139 while True:
1140 trisycl_include_dir = get_from_env_or_user_or_default(
Michael Cased31531a2018-01-05 14:09:41 -08001141 environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir,
1142 _DEFAULT_TRISYCL_INCLUDE_DIR)
Dandelion Man?90e42f32017-12-15 18:15:07 -08001143 if os.path.exists(trisycl_include_dir):
1144 break
1145
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001146 print('Invalid triSYCL include directory, %s cannot be found' %
1147 (trisycl_include_dir))
Dandelion Man?90e42f32017-12-15 18:15:07 -08001148
1149 # Set TRISYCL_INCLUDE_DIR
1150 environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001151 write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', trisycl_include_dir)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001152
Yifei Fengb1d8c592017-11-22 13:42:21 -08001153
A. Unique TensorFlowere9797fd2019-03-22 11:02:40 -07001154def system_specific_test_config(env):
A. Unique TensorFlower7bd86372019-03-21 15:19:30 -07001155 """Add default build and test flags required for TF tests to bazelrc."""
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001156 write_to_bazelrc('test --flaky_test_attempts=3')
1157 write_to_bazelrc('test --test_size_filters=small,medium')
frreiss8b013a72019-11-20 11:36:27 -08001158
1159 # Each instance of --test_tag_filters or --build_tag_filters overrides all
1160 # previous instances, so we need to build up a complete list and write a
1161 # single list of filters for the .bazelrc file.
1162
1163 # Filters to use with both --test_tag_filters and --build_tag_filters
frreissbab7d7d2019-11-21 10:52:55 -08001164 test_and_build_filters = ['-benchmark-test', '-no_oss']
1165 # Additional filters for --test_tag_filters beyond those in
1166 # test_and_build_filters
1167 test_only_filters = ['-oss_serial']
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001168 if is_windows():
frreissbab7d7d2019-11-21 10:52:55 -08001169 test_and_build_filters.append('-no_windows')
Guangda Laibcd701a2019-03-12 21:04:51 -07001170 if env.get('TF_NEED_CUDA', None) == '1':
frreissbab7d7d2019-11-21 10:52:55 -08001171 test_and_build_filters += ['-no_windows_gpu', '-no_gpu']
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001172 else:
frreissbab7d7d2019-11-21 10:52:55 -08001173 test_and_build_filters.append('-gpu')
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001174 elif is_macos():
frreissbab7d7d2019-11-21 10:52:55 -08001175 test_and_build_filters += ['-gpu', '-nomac', '-no_mac']
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001176 elif is_linux():
Guangda Laibcd701a2019-03-12 21:04:51 -07001177 if env.get('TF_NEED_CUDA', None) == '1':
frreissbab7d7d2019-11-21 10:52:55 -08001178 test_and_build_filters.append('-no_gpu')
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001179 write_to_bazelrc('test --test_env=LD_LIBRARY_PATH')
1180 else:
frreissbab7d7d2019-11-21 10:52:55 -08001181 test_and_build_filters.append('-gpu')
frreiss3adf4172019-12-06 14:16:49 -08001182
1183 # Disable tests with "v1only" tag in "v2" Bazel config, but not in "v1" config
1184 write_to_bazelrc('test:v1 --test_tag_filters=%s' %
TensorFlower Gardenere5159082019-11-22 11:47:38 -08001185 ','.join(test_and_build_filters + test_only_filters))
frreiss3adf4172019-12-06 14:16:49 -08001186 write_to_bazelrc('test:v1 --build_tag_filters=%s' %
TensorFlower Gardenere5159082019-11-22 11:47:38 -08001187 ','.join(test_and_build_filters))
frreiss3adf4172019-12-06 14:16:49 -08001188 write_to_bazelrc('test:v2 --test_tag_filters=%s' %
1189 ','.join(test_and_build_filters + test_only_filters
1190 + ["-v1only"]))
1191 write_to_bazelrc('test:v2 --build_tag_filters=%s' %
1192 ','.join(test_and_build_filters + ["-v1only"]))
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001193
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001194
Yifei Feng5198cb82018-08-17 13:53:06 -07001195def set_system_libs_flag(environ_cp):
1196 syslibs = environ_cp.get('TF_SYSTEM_LIBS', '')
TensorFlower Gardener61a87202018-10-01 12:25:39 -07001197 if syslibs:
Jason Zaman5fc39bd2018-09-16 01:38:55 +08001198 if ',' in syslibs:
1199 syslibs = ','.join(sorted(syslibs.split(',')))
1200 else:
1201 syslibs = ','.join(sorted(syslibs.split()))
Yifei Feng5198cb82018-08-17 13:53:06 -07001202 write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs)
1203
Jason Zaman5fc39bd2018-09-16 01:38:55 +08001204 if 'PREFIX' in environ_cp:
1205 write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX'])
1206 if 'LIBDIR' in environ_cp:
1207 write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR'])
1208 if 'INCLUDEDIR' in environ_cp:
1209 write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR'])
1210
Yifei Feng5198cb82018-08-17 13:53:06 -07001211
Austin Anderson6d7c3762019-11-19 17:04:19 -08001212def is_reduced_optimize_huge_functions_available(environ_cp):
1213 """Check to see if the system supports /d2ReducedOptimizeHugeFunctions.
1214
1215 The above compiler flag is a new compiler flag introduced to the Visual Studio
1216 compiler in version 16.4 (available in Visual Studio 2019, Preview edition
1217 only, as of 2019-11-19). TensorFlow needs this flag to massively reduce
1218 compile times, but until 16.4 is officially released, we can't depend on it.
1219
1220 See also https://groups.google.com/a/tensorflow.org/g/build/c/SsW98Eo7l3o
1221
1222 Because it's very annoying to check this manually (to check the MSVC installed
1223 versions, you need to use the registry, and it's not clear if Bazel will be
1224 using that install version anyway), we expect enviroments who know they may
1225 use this flag to export TF_VC_VERSION=16.4
1226
1227 TODO(angerson, gunan): Remove this function when TensorFlow's minimum VS
1228 version is upgraded to 16.4.
1229
1230 Arguments:
1231 environ_cp: Environment of the current execution
1232
1233 Returns:
1234 boolean, whether or not /d2ReducedOptimizeHugeFunctions is available on this
1235 machine.
1236 """
1237 return float(environ_cp.get('TF_VC_VERSION', '0')) >= 16.4
1238
1239
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001240def set_windows_build_flags(environ_cp):
1241 """Set Windows specific build options."""
Austin Anderson6d7c3762019-11-19 17:04:19 -08001242 if is_reduced_optimize_huge_functions_available(environ_cp):
1243 write_to_bazelrc(
1244 'build --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions'
1245 )
1246
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001247 if get_var(
1248 environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline',
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001249 True, ('Would you like to override eigen strong inline for some C++ '
1250 'compilation to reduce the compilation time?'),
1251 'Eigen strong inline overridden.', 'Not overriding eigen strong inline, '
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001252 'some compilations could take more than 20 mins.'):
1253 # Due to a known MSVC compiler issue
1254 # https://github.com/tensorflow/tensorflow/issues/10521
1255 # Overriding eigen strong inline speeds up the compiling of
1256 # conv_grad_ops_3d.cc and conv_ops_3d.cc by 20 minutes,
1257 # but this also hurts the performance. Let users decide what they want.
1258 write_to_bazelrc('build --define=override_eigen_strong_inline=true')
Dandelion Man?90e42f32017-12-15 18:15:07 -08001259
A. Unique TensorFlower061c3592017-11-13 14:21:04 -08001260
Michael Cased31531a2018-01-05 14:09:41 -08001261def config_info_line(name, help_text):
1262 """Helper function to print formatted help text for Bazel config options."""
1263 print('\t--config=%-12s\t# %s' % (name, help_text))
1264
1265
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001266def configure_ios():
1267 """Configures TensorFlow for iOS builds.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001268
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001269 This function will only be executed if `is_macos()` is true.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001270 """
1271 if not is_macos():
1272 return
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001273 for filepath in APPLE_BAZEL_FILES:
1274 existing_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath + '.apple')
1275 renamed_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath)
1276 symlink_force(existing_filepath, renamed_filepath)
1277 for filepath in IOS_FILES:
1278 filename = os.path.basename(filepath)
1279 new_filepath = os.path.join(_TF_WORKSPACE_ROOT, filename)
1280 symlink_force(filepath, new_filepath)
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001281
1282
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001283def validate_cuda_config(environ_cp):
1284 """Run find_cuda_config.py and return cuda_toolkit_path, or None."""
1285
1286 def maybe_encode_env(env):
1287 """Encodes unicode in env to str on Windows python 2.x."""
1288 if not is_windows() or sys.version_info[0] != 2:
1289 return env
1290 for k, v in env.items():
1291 if isinstance(k, unicode):
1292 k = k.encode('ascii')
1293 if isinstance(v, unicode):
1294 v = v.encode('ascii')
1295 env[k] = v
1296 return env
1297
1298 cuda_libraries = ['cuda', 'cudnn']
1299 if is_linux():
A. Unique TensorFlower28fc9cc2019-05-01 14:17:54 -07001300 if int(environ_cp.get('TF_NEED_TENSORRT', False)):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001301 cuda_libraries.append('tensorrt')
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001302 if environ_cp.get('TF_NCCL_VERSION', None):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001303 cuda_libraries.append('nccl')
1304
1305 proc = subprocess.Popen(
1306 [environ_cp['PYTHON_BIN_PATH'], 'third_party/gpus/find_cuda_config.py'] +
1307 cuda_libraries,
1308 stdout=subprocess.PIPE,
1309 env=maybe_encode_env(environ_cp))
1310
1311 if proc.wait():
1312 # Errors from find_cuda_config.py were sent to stderr.
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -07001313 print('Asking for detailed CUDA configuration...\n')
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001314 return False
1315
1316 config = dict(
1317 tuple(line.decode('ascii').rstrip().split(': ')) for line in proc.stdout)
1318
1319 print('Found CUDA %s in:' % config['cuda_version'])
1320 print(' %s' % config['cuda_library_dir'])
1321 print(' %s' % config['cuda_include_dir'])
1322
1323 print('Found cuDNN %s in:' % config['cudnn_version'])
1324 print(' %s' % config['cudnn_library_dir'])
1325 print(' %s' % config['cudnn_include_dir'])
1326
1327 if 'tensorrt_version' in config:
1328 print('Found TensorRT %s in:' % config['tensorrt_version'])
1329 print(' %s' % config['tensorrt_library_dir'])
1330 print(' %s' % config['tensorrt_include_dir'])
1331
1332 if config.get('nccl_version', None):
1333 print('Found NCCL %s in:' % config['nccl_version'])
1334 print(' %s' % config['nccl_library_dir'])
1335 print(' %s' % config['nccl_include_dir'])
1336
1337 print('\n')
1338
1339 environ_cp['CUDA_TOOLKIT_PATH'] = config['cuda_toolkit_path']
1340 return True
1341
1342
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001343def main():
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001344 global _TF_WORKSPACE_ROOT
1345 global _TF_BAZELRC
A. Unique TensorFlowered297342019-03-15 11:25:28 -07001346 global _TF_CURRENT_BAZEL_VERSION
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001347
Shanqing Cai71445712018-03-12 19:33:52 -07001348 parser = argparse.ArgumentParser()
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001349 parser.add_argument(
1350 '--workspace',
1351 type=str,
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001352 default=os.path.abspath(os.path.dirname(__file__)),
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001353 help='The absolute path to your active Bazel workspace.')
Shanqing Cai71445712018-03-12 19:33:52 -07001354 args = parser.parse_args()
1355
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001356 _TF_WORKSPACE_ROOT = args.workspace
1357 _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME)
1358
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001359 # Make a copy of os.environ to be clear when functions and getting and setting
1360 # environment variables.
1361 environ_cp = dict(os.environ)
1362
Mark Daoust44000ad2019-06-18 09:26:26 -07001363 current_bazel_version = check_bazel_version(_TF_MIN_BAZEL_VERSION,
1364 _TF_MAX_BAZEL_VERSION)
A. Unique TensorFlowered297342019-03-15 11:25:28 -07001365 _TF_CURRENT_BAZEL_VERSION = convert_version_to_int(current_bazel_version)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001366
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001367 reset_tf_configure_bazelrc()
Yun Peng03e63a22018-11-07 11:18:53 +01001368
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001369 cleanup_makefile()
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -07001370 setup_python(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001371
1372 if is_windows():
Yifei Fengb1d8c592017-11-22 13:42:21 -08001373 environ_cp['TF_NEED_OPENCL_SYCL'] = '0'
1374 environ_cp['TF_NEED_COMPUTECPP'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001375 environ_cp['TF_NEED_OPENCL'] = '0'
1376 environ_cp['TF_CUDA_CLANG'] = '0'
Guangda Lai76f69382018-01-25 23:59:19 -08001377 environ_cp['TF_NEED_TENSORRT'] = '0'
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001378 # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on
1379 # Windows.
1380 environ_cp['TF_DOWNLOAD_CLANG'] = '0'
A. Unique TensorFlower6e97fb32018-07-16 14:07:29 -07001381 environ_cp['TF_NEED_MPI'] = '0'
1382 environ_cp['TF_SET_ANDROID_WORKSPACE'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001383
1384 if is_macos():
Guangda Lai76f69382018-01-25 23:59:19 -08001385 environ_cp['TF_NEED_TENSORRT'] = '0'
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001386 else:
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001387 environ_cp['TF_CONFIGURE_IOS'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001388
Sanjoy Dasf3fab7c2019-06-20 11:19:00 -07001389 xla_enabled_by_default = is_linux() or is_macos()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001390 set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
Grzegorz Pawelczakec82efd2018-10-09 15:03:46 +01001391 xla_enabled_by_default, 'xla')
Gunhan Gulsoy3da0dff2018-09-26 11:55:50 -07001392
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001393 set_action_env_var(
1394 environ_cp,
1395 'TF_NEED_OPENCL_SYCL',
1396 'OpenCL SYCL',
1397 False,
1398 bazel_config_name='sycl')
Yifei Fengb1d8c592017-11-22 13:42:21 -08001399 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1':
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001400 set_host_cxx_compiler(environ_cp)
1401 set_host_c_compiler(environ_cp)
Yifei Fengb1d8c592017-11-22 13:42:21 -08001402 set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True)
1403 if environ_cp.get('TF_NEED_COMPUTECPP') == '1':
1404 set_computecpp_toolkit_path(environ_cp)
1405 else:
1406 set_trisycl_include_dir(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001407
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001408 set_action_env_var(
1409 environ_cp, 'TF_NEED_ROCM', 'ROCm', False, bazel_config_name='rocm')
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001410 if (environ_cp.get('TF_NEED_ROCM') == '1' and
TensorFlower Gardener62e60162018-09-27 10:22:55 -07001411 'LD_LIBRARY_PATH' in environ_cp and
1412 environ_cp.get('LD_LIBRARY_PATH') != '1'):
1413 write_action_env_to_bazelrc('LD_LIBRARY_PATH',
1414 environ_cp.get('LD_LIBRARY_PATH'))
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001415
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001416 environ_cp['TF_NEED_CUDA'] = str(
1417 int(get_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False)))
A. Unique TensorFlower24cbb2a2017-09-08 07:45:44 -07001418 if (environ_cp.get('TF_NEED_CUDA') == '1' and
1419 'TF_CUDA_CONFIG_REPO' not in environ_cp):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001420
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001421 set_action_env_var(
1422 environ_cp,
1423 'TF_NEED_TENSORRT',
1424 'TensorRT',
1425 False,
1426 bazel_config_name='tensorrt')
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001427
1428 environ_save = dict(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001429 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001430
1431 if validate_cuda_config(environ_cp):
1432 cuda_env_names = [
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001433 'TF_CUDA_VERSION',
1434 'TF_CUBLAS_VERSION',
1435 'TF_CUDNN_VERSION',
1436 'TF_TENSORRT_VERSION',
1437 'TF_NCCL_VERSION',
1438 'TF_CUDA_PATHS',
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001439 # Items below are for backwards compatibility when not using
1440 # TF_CUDA_PATHS.
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001441 'CUDA_TOOLKIT_PATH',
1442 'CUDNN_INSTALL_PATH',
1443 'NCCL_INSTALL_PATH',
1444 'NCCL_HDR_PATH',
1445 'TENSORRT_INSTALL_PATH'
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001446 ]
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001447 # Note: set_action_env_var above already writes to bazelrc.
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001448 for name in cuda_env_names:
1449 if name in environ_cp:
1450 write_action_env_to_bazelrc(name, environ_cp[name])
1451 break
1452
1453 # Restore settings changed below if CUDA config could not be validated.
1454 environ_cp = dict(environ_save)
1455
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001456 set_tf_cuda_version(environ_cp)
1457 set_tf_cudnn_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001458 if is_linux():
1459 set_tf_tensorrt_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001460 set_tf_nccl_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001461
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001462 set_tf_cuda_paths(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001463
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001464 else:
1465 raise UserInputError(
1466 'Invalid CUDA setting were provided %d '
1467 'times in a row. Assuming to be a scripting mistake.' %
1468 _DEFAULT_PROMPT_ASK_ATTEMPTS)
1469
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001470 set_tf_cuda_compute_capabilities(environ_cp)
Ankur Taly0e6f39d2018-02-16 18:22:55 -08001471 if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get(
1472 'LD_LIBRARY_PATH') != '1':
1473 write_action_env_to_bazelrc('LD_LIBRARY_PATH',
1474 environ_cp.get('LD_LIBRARY_PATH'))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001475
1476 set_tf_cuda_clang(environ_cp)
1477 if environ_cp.get('TF_CUDA_CLANG') == '1':
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001478 # Ask whether we should download the clang toolchain.
1479 set_tf_download_clang(environ_cp)
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -08001480 if environ_cp.get('TF_DOWNLOAD_CLANG') != '1':
1481 # Set up which clang we should use as the cuda / host compiler.
1482 set_clang_cuda_compiler_path(environ_cp)
Ilya Biryukov1c3d02e2018-09-04 03:09:52 -07001483 else:
1484 # Use downloaded LLD for linking.
1485 write_to_bazelrc('build:cuda_clang --config=download_clang_use_lld')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001486 else:
1487 # Set up which gcc nvcc should use as the host compiler
1488 # No need to set this on Windows
1489 if not is_windows():
1490 set_gcc_host_compiler_path(environ_cp)
1491 set_other_cuda_vars(environ_cp)
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001492 else:
1493 # CUDA not required. Ask whether we should download the clang toolchain and
1494 # use it for the CPU build.
1495 set_tf_download_clang(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001496
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001497 # SYCL / ROCm / CUDA are mutually exclusive.
1498 # At most 1 GPU platform can be configured.
1499 gpu_platform_count = 0
1500 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1':
1501 gpu_platform_count += 1
1502 if environ_cp.get('TF_NEED_ROCM') == '1':
1503 gpu_platform_count += 1
1504 if environ_cp.get('TF_NEED_CUDA') == '1':
1505 gpu_platform_count += 1
1506 if gpu_platform_count >= 2:
1507 raise UserInputError('SYCL / CUDA / ROCm are mututally exclusive. '
1508 'At most 1 GPU platform can be configured.')
1509
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001510 set_cc_opt_flags(environ_cp)
Yifei Feng5198cb82018-08-17 13:53:06 -07001511 set_system_libs_flag(environ_cp)
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001512 if is_windows():
1513 set_windows_build_flags(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001514
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001515 if get_var(environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', False,
1516 ('Would you like to interactively configure ./WORKSPACE for '
1517 'Android builds?'), 'Searching for NDK and SDK installations.',
1518 'Not configuring the WORKSPACE for Android builds.'):
Michael Case51053502018-06-05 17:47:19 -07001519 create_android_ndk_rule(environ_cp)
1520 create_android_sdk_rule(environ_cp)
Austin Anderson6afface2017-12-05 11:59:17 -08001521
A. Unique TensorFlowere9797fd2019-03-22 11:02:40 -07001522 system_specific_test_config(os.environ)
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001523
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -07001524 set_action_env_var(environ_cp, 'TF_CONFIGURE_IOS', 'iOS', False)
1525 if environ_cp.get('TF_CONFIGURE_IOS') == '1':
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001526 configure_ios()
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001527
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001528 print('Preconfigured Bazel build configs. You can use any of the below by '
1529 'adding "--config=<>" to your build command. See .bazelrc for more '
1530 'details.')
1531 config_info_line('mkl', 'Build with MKL support.')
1532 config_info_line('monolithic', 'Config for mostly static monolithic build.')
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001533 config_info_line('ngraph', 'Build with Intel nGraph support.')
A. Unique TensorFlowera6bf9c82019-02-26 10:08:35 -08001534 config_info_line('numa', 'Build with NUMA support.')
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -08001535 config_info_line(
1536 'dynamic_kernels',
1537 '(Experimental) Build kernels into separate shared objects.')
Anna Raedf6742019-06-12 11:30:51 -07001538 config_info_line('v2', 'Build TensorFlow 2.x instead of 1.x.')
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001539
1540 print('Preconfigured Bazel build configs to DISABLE default on features:')
1541 config_info_line('noaws', 'Disable AWS S3 filesystem support.')
1542 config_info_line('nogcp', 'Disable GCP support.')
1543 config_info_line('nohdfs', 'Disable HDFS support.')
Gunhan Gulsoyeea81682018-11-26 16:51:23 -08001544 config_info_line('nonccl', 'Disable NVIDIA NCCL support.')
Austin Anderson6afface2017-12-05 11:59:17 -08001545
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001546
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001547if __name__ == '__main__':
1548 main()