blob: 46ce13ea92faecc5892500e07471d3e207d45dc2 [file] [log] [blame]
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""configure script to get build parameters from user."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
Shanqing Cai71445712018-03-12 19:33:52 -070021import argparse
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070022import errno
23import os
24import platform
25import re
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070026import subprocess
27import sys
28
Andrew Sellec9885ea2017-11-06 09:37:03 -080029# pylint: disable=g-import-not-at-top
Jonathan Hseu008910f2017-08-25 14:01:05 -070030try:
31 from shutil import which
32except ImportError:
33 from distutils.spawn import find_executable as which
Andrew Sellec9885ea2017-11-06 09:37:03 -080034# pylint: enable=g-import-not-at-top
Jonathan Hseu008910f2017-08-25 14:01:05 -070035
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -070036_DEFAULT_CUDA_VERSION = '10'
37_DEFAULT_CUDNN_VERSION = '7'
Guangda Laifaa93ac2019-10-23 15:07:12 -070038_DEFAULT_TENSORRT_VERSION = '6'
Smit Hinsufe7d1d92018-07-14 13:16:58 -070039_DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0'
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -070040
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070041_TF_OPENCL_VERSION = '1.2'
42_DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp'
Yifei Fengb1d8c592017-11-22 13:42:21 -080043_DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include'
A. Unique TensorFlower82820ef2018-11-12 13:22:13 -080044_SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15, 16, 17, 18]
Austin Anderson6afface2017-12-05 11:59:17 -080045
46_DEFAULT_PROMPT_ASK_ATTEMPTS = 10
47
Shanqing Cai71445712018-03-12 19:33:52 -070048_TF_BAZELRC_FILENAME = '.tf_configure.bazelrc'
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -070049_TF_WORKSPACE_ROOT = ''
50_TF_BAZELRC = ''
A. Unique TensorFlowered297342019-03-15 11:25:28 -070051_TF_CURRENT_BAZEL_VERSION = None
Mark Daoust44000ad2019-06-18 09:26:26 -070052_TF_MIN_BAZEL_VERSION = '0.24.1'
A. Unique TensorFlowercb677d12019-10-18 06:32:18 -070053_TF_MAX_BAZEL_VERSION = '0.29.1'
Shanqing Cai71445712018-03-12 19:33:52 -070054
Jason Furmanek7c234152018-09-26 04:44:12 +000055NCCL_LIB_PATHS = [
TensorFlower Gardenerea5c5292018-10-01 17:28:26 -070056 'lib64/', 'lib/powerpc64le-linux-gnu/', 'lib/x86_64-linux-gnu/', ''
Jason Furmanek7c234152018-09-26 04:44:12 +000057]
Austin Anderson6afface2017-12-05 11:59:17 -080058
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -070059# List of files to configure when building Bazel on Apple platforms.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -080060APPLE_BAZEL_FILES = [
A. Unique TensorFlower6a059c02019-04-05 14:28:27 -070061 'tensorflow/lite/experimental/ios/BUILD',
A. Unique TensorFlower93e70732019-02-14 16:45:32 -080062 'tensorflow/lite/experimental/objc/BUILD',
63 'tensorflow/lite/experimental/swift/BUILD'
64]
65
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -070066# List of files to move when building for iOS.
67IOS_FILES = [
68 'tensorflow/lite/experimental/objc/TensorFlowLiteObjC.podspec',
69 'tensorflow/lite/experimental/swift/TensorFlowLiteSwift.podspec',
70]
71
Austin Anderson6afface2017-12-05 11:59:17 -080072
73class UserInputError(Exception):
74 pass
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070075
76
77def is_windows():
78 return platform.system() == 'Windows'
79
80
81def is_linux():
82 return platform.system() == 'Linux'
83
84
85def is_macos():
86 return platform.system() == 'Darwin'
87
88
89def is_ppc64le():
90 return platform.machine() == 'ppc64le'
91
92
Jonathan Hseu008910f2017-08-25 14:01:05 -070093def is_cygwin():
94 return platform.system().startswith('CYGWIN_NT')
95
96
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070097def get_input(question):
98 try:
99 try:
100 answer = raw_input(question)
101 except NameError:
102 answer = input(question) # pylint: disable=bad-builtin
103 except EOFError:
104 answer = ''
105 return answer
106
107
108def symlink_force(target, link_name):
109 """Force symlink, equivalent of 'ln -sf'.
110
111 Args:
112 target: items to link to.
113 link_name: name of the link.
114 """
115 try:
116 os.symlink(target, link_name)
117 except OSError as e:
118 if e.errno == errno.EEXIST:
119 os.remove(link_name)
120 os.symlink(target, link_name)
121 else:
122 raise e
123
124
125def sed_in_place(filename, old, new):
126 """Replace old string with new string in file.
127
128 Args:
129 filename: string for filename.
130 old: string to replace.
131 new: new string to replace to.
132 """
133 with open(filename, 'r') as f:
134 filedata = f.read()
135 newdata = filedata.replace(old, new)
136 with open(filename, 'w') as f:
137 f.write(newdata)
138
139
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700140def write_to_bazelrc(line):
141 with open(_TF_BAZELRC, 'a') as f:
142 f.write(line + '\n')
143
144
145def write_action_env_to_bazelrc(var_name, var):
146 write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var)))
147
148
Jonathan Hseu008910f2017-08-25 14:01:05 -0700149def run_shell(cmd, allow_non_zero=False):
150 if allow_non_zero:
151 try:
152 output = subprocess.check_output(cmd)
153 except subprocess.CalledProcessError as e:
154 output = e.output
155 else:
156 output = subprocess.check_output(cmd)
157 return output.decode('UTF-8').strip()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700158
159
160def cygpath(path):
161 """Convert path from posix to windows."""
Martin Wicked57572e2017-09-02 19:21:45 -0700162 return os.path.abspath(path).replace('\\', '/')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700163
164
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700165def get_python_path(environ_cp, python_bin_path):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700166 """Get the python site package paths."""
167 python_paths = []
168 if environ_cp.get('PYTHONPATH'):
169 python_paths = environ_cp.get('PYTHONPATH').split(':')
170 try:
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700171 library_paths = run_shell([
172 python_bin_path, '-c',
173 'import site; print("\\n".join(site.getsitepackages()))'
174 ]).split('\n')
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700175 except subprocess.CalledProcessError:
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700176 library_paths = [
177 run_shell([
178 python_bin_path, '-c',
179 'from distutils.sysconfig import get_python_lib;'
180 'print(get_python_lib())'
181 ])
182 ]
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700183
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700184 all_paths = set(python_paths + library_paths)
185
186 paths = []
187 for path in all_paths:
188 if os.path.isdir(path):
189 paths.append(path)
190 return paths
191
192
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700193def get_python_major_version(python_bin_path):
194 """Get the python major version."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700195 return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])'])
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700196
197
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -0700198def setup_python(environ_cp):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700199 """Setup python related env variables."""
200 # Get PYTHON_BIN_PATH, default is the current running python.
201 default_python_bin_path = sys.executable
202 ask_python_bin_path = ('Please specify the location of python. [Default is '
203 '%s]: ') % default_python_bin_path
204 while True:
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700205 python_bin_path = get_from_env_or_user_or_default(environ_cp,
206 'PYTHON_BIN_PATH',
207 ask_python_bin_path,
208 default_python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700209 # Check if the path is valid
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700210 if os.path.isfile(python_bin_path) and os.access(python_bin_path, os.X_OK):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700211 break
212 elif not os.path.exists(python_bin_path):
213 print('Invalid python path: %s cannot be found.' % python_bin_path)
214 else:
215 print('%s is not executable. Is it the python binary?' % python_bin_path)
216 environ_cp['PYTHON_BIN_PATH'] = ''
217
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700218 # Convert python path to Windows style before checking lib and version
Martin Wicked57572e2017-09-02 19:21:45 -0700219 if is_windows() or is_cygwin():
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700220 python_bin_path = cygpath(python_bin_path)
221
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700222 # Get PYTHON_LIB_PATH
223 python_lib_path = environ_cp.get('PYTHON_LIB_PATH')
224 if not python_lib_path:
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700225 python_lib_paths = get_python_path(environ_cp, python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700226 if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1':
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700227 python_lib_path = python_lib_paths[0]
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700228 else:
Jonathan Hseu008910f2017-08-25 14:01:05 -0700229 print('Found possible Python library paths:\n %s' %
230 '\n '.join(python_lib_paths))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700231 default_python_lib_path = python_lib_paths[0]
232 python_lib_path = get_input(
Jonathan Hseu008910f2017-08-25 14:01:05 -0700233 'Please input the desired Python library path to use. '
234 'Default is [%s]\n' % python_lib_paths[0])
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700235 if not python_lib_path:
236 python_lib_path = default_python_lib_path
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700237 environ_cp['PYTHON_LIB_PATH'] = python_lib_path
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700238
A. Unique TensorFlowercfccbdb2019-06-24 11:01:58 -0700239 python_major_version = get_python_major_version(python_bin_path)
240 if python_major_version == '2':
241 write_to_bazelrc('build --host_force_python=PY2')
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700242
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700243 # Convert python path to Windows style before writing into bazel.rc
Martin Wicked57572e2017-09-02 19:21:45 -0700244 if is_windows() or is_cygwin():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700245 python_lib_path = cygpath(python_lib_path)
246
247 # Set-up env variables used by python_configure.bzl
248 write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path)
249 write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path)
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -0700250 write_to_bazelrc('build --python_path=\"%s"' % python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700251 environ_cp['PYTHON_BIN_PATH'] = python_bin_path
252
William D. Ironsdcc76a52018-11-20 10:35:18 -0600253 # If choosen python_lib_path is from a path specified in the PYTHONPATH
254 # variable, need to tell bazel to include PYTHONPATH
255 if environ_cp.get('PYTHONPATH'):
256 python_paths = environ_cp.get('PYTHONPATH').split(':')
257 if python_lib_path in python_paths:
TensorFlower Gardener968cd182018-11-28 11:33:16 -0800258 write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH'))
William D. Ironsdcc76a52018-11-20 10:35:18 -0600259
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700260 # Write tools/python_bin_path.sh
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700261 with open(
262 os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'),
263 'w') as f:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700264 f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path)
265
266
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -0700267def reset_tf_configure_bazelrc():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700268 """Reset file that contains customized config settings."""
269 open(_TF_BAZELRC, 'w').close()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700270
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800271
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700272def cleanup_makefile():
273 """Delete any leftover BUILD files from the Makefile build.
274
275 These files could interfere with Bazel parsing.
276 """
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700277 makefile_download_dir = os.path.join(_TF_WORKSPACE_ROOT, 'tensorflow',
278 'contrib', 'makefile', 'downloads')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700279 if os.path.isdir(makefile_download_dir):
280 for root, _, filenames in os.walk(makefile_download_dir):
281 for f in filenames:
282 if f.endswith('BUILD'):
283 os.remove(os.path.join(root, f))
284
285
286def get_var(environ_cp,
287 var_name,
288 query_item,
289 enabled_by_default,
290 question=None,
291 yes_reply=None,
292 no_reply=None):
293 """Get boolean input from user.
294
295 If var_name is not set in env, ask user to enable query_item or not. If the
296 response is empty, use the default.
297
298 Args:
299 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530300 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
301 query_item: string for feature related to the variable, e.g. "CUDA for
302 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700303 enabled_by_default: boolean for default behavior.
304 question: optional string for how to ask for user input.
Michael Cased90054e2018-02-07 14:36:00 -0800305 yes_reply: optional string for reply when feature is enabled.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700306 no_reply: optional string for reply when feature is disabled.
307
308 Returns:
309 boolean value of the variable.
Frank Chenc4ef9272018-01-10 11:36:52 -0800310
311 Raises:
312 UserInputError: if an environment variable is set, but it cannot be
313 interpreted as a boolean indicator, assume that the user has made a
314 scripting error, and will continue to provide invalid input.
315 Raise the error to avoid infinitely looping.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700316 """
317 if not question:
318 question = 'Do you wish to build TensorFlow with %s support?' % query_item
319 if not yes_reply:
320 yes_reply = '%s support will be enabled for TensorFlow.' % query_item
321 if not no_reply:
322 no_reply = 'No %s' % yes_reply
323
324 yes_reply += '\n'
325 no_reply += '\n'
326
327 if enabled_by_default:
328 question += ' [Y/n]: '
329 else:
330 question += ' [y/N]: '
331
332 var = environ_cp.get(var_name)
Frank Chenc4ef9272018-01-10 11:36:52 -0800333 if var is not None:
334 var_content = var.strip().lower()
335 true_strings = ('1', 't', 'true', 'y', 'yes')
336 false_strings = ('0', 'f', 'false', 'n', 'no')
337 if var_content in true_strings:
338 var = True
339 elif var_content in false_strings:
340 var = False
341 else:
342 raise UserInputError(
343 'Environment variable %s must be set as a boolean indicator.\n'
344 'The following are accepted as TRUE : %s.\n'
345 'The following are accepted as FALSE: %s.\n'
A. Unique TensorFlowered297342019-03-15 11:25:28 -0700346 'Current value is %s.' %
347 (var_name, ', '.join(true_strings), ', '.join(false_strings), var))
Frank Chenc4ef9272018-01-10 11:36:52 -0800348
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700349 while var is None:
350 user_input_origin = get_input(question)
351 user_input = user_input_origin.strip().lower()
352 if user_input == 'y':
353 print(yes_reply)
354 var = True
355 elif user_input == 'n':
356 print(no_reply)
357 var = False
358 elif not user_input:
359 if enabled_by_default:
360 print(yes_reply)
361 var = True
362 else:
363 print(no_reply)
364 var = False
365 else:
366 print('Invalid selection: %s' % user_input_origin)
367 return var
368
369
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700370def set_build_var(environ_cp,
371 var_name,
372 query_item,
373 option_name,
374 enabled_by_default,
375 bazel_config_name=None):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700376 """Set if query_item will be enabled for the build.
377
378 Ask user if query_item will be enabled. Default is used if no input is given.
379 Set subprocess environment variable and write to .bazelrc if enabled.
380
381 Args:
382 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530383 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
384 query_item: string for feature related to the variable, e.g. "CUDA for
385 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700386 option_name: string for option to define in .bazelrc.
387 enabled_by_default: boolean for default behavior.
Michael Case98850a52017-09-14 13:35:57 -0700388 bazel_config_name: Name for Bazel --config argument to enable build feature.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700389 """
390
391 var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default)))
392 environ_cp[var_name] = var
393 if var == '1':
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700394 write_to_bazelrc('build:%s --define %s=true' %
395 (bazel_config_name, option_name))
Yifei Fengec451f52018-10-05 12:53:50 -0700396 write_to_bazelrc('build --config=%s' % bazel_config_name)
Michael Case98850a52017-09-14 13:35:57 -0700397 elif bazel_config_name is not None:
398 # TODO(mikecase): Migrate all users of configure.py to use --config Bazel
399 # options and not to set build configs through environment variables.
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700400 write_to_bazelrc('build:%s --define %s=true' %
401 (bazel_config_name, option_name))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700402
403
404def set_action_env_var(environ_cp,
405 var_name,
406 query_item,
407 enabled_by_default,
408 question=None,
409 yes_reply=None,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700410 no_reply=None,
411 bazel_config_name=None):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700412 """Set boolean action_env variable.
413
414 Ask user if query_item will be enabled. Default is used if no input is given.
415 Set environment variable and write to .bazelrc.
416
417 Args:
418 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530419 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
420 query_item: string for feature related to the variable, e.g. "CUDA for
421 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700422 enabled_by_default: boolean for default behavior.
423 question: optional string for how to ask for user input.
Michael Cased90054e2018-02-07 14:36:00 -0800424 yes_reply: optional string for reply when feature is enabled.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700425 no_reply: optional string for reply when feature is disabled.
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700426 bazel_config_name: adding config to .bazelrc instead of action_env.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700427 """
428 var = int(
429 get_var(environ_cp, var_name, query_item, enabled_by_default, question,
430 yes_reply, no_reply))
431
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700432 if not bazel_config_name:
433 write_action_env_to_bazelrc(var_name, var)
434 elif var:
435 write_to_bazelrc('build --config=%s' % bazel_config_name)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700436 environ_cp[var_name] = str(var)
437
438
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700439def convert_version_to_int(version):
440 """Convert a version number to a integer that can be used to compare.
441
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700442 Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The
443 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored.
444
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700445 Args:
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700446 version: a version to be converted
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700447
448 Returns:
449 An integer if converted successfully, otherwise return None.
450 """
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700451 version = version.split('-')[0]
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700452 version_segments = version.split('.')
Austin Anderson87ea41d2019-04-04 10:03:50 -0700453 # Treat "0.24" as "0.24.0"
454 if len(version_segments) == 2:
455 version_segments.append('0')
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700456 for seg in version_segments:
457 if not seg.isdigit():
458 return None
459
460 version_str = ''.join(['%03d' % int(seg) for seg in version_segments])
461 return int(version_str)
462
463
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800464def check_bazel_version(min_version, max_version):
465 """Check installed bazel version is between min_version and max_version.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700466
467 Args:
Mihai Maruseac4db860f2019-04-19 08:52:10 -0700468 min_version: string for minimum bazel version (must exist!).
469 max_version: string for maximum bazel version (must exist!).
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700470
471 Returns:
472 The bazel version detected.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700473 """
Jonathan Hseu008910f2017-08-25 14:01:05 -0700474 if which('bazel') is None:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700475 print('Cannot find bazel. Please install bazel.')
476 sys.exit(0)
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700477 curr_version = run_shell(
478 ['bazel', '--batch', '--bazelrc=/dev/null', 'version'])
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700479
480 for line in curr_version.split('\n'):
481 if 'Build label: ' in line:
482 curr_version = line.split('Build label: ')[1]
483 break
484
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700485 min_version_int = convert_version_to_int(min_version)
486 curr_version_int = convert_version_to_int(curr_version)
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800487 max_version_int = convert_version_to_int(max_version)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700488
489 # Check if current bazel version can be detected properly.
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700490 if not curr_version_int:
491 print('WARNING: current bazel installation is not a release version.')
492 print('Make sure you are running at least bazel %s' % min_version)
493 return curr_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700494
Michael Cased94271a2017-08-22 17:26:52 -0700495 print('You have bazel %s installed.' % curr_version)
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700496
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700497 if curr_version_int < min_version_int:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700498 print('Please upgrade your bazel installation to version %s or higher to '
499 'build TensorFlow!' % min_version)
Jason Zamanb41761c2018-10-14 11:28:53 +0800500 sys.exit(1)
TensorFlower Gardener78c246b2018-12-13 12:37:42 -0800501 if (curr_version_int > max_version_int and
502 'TF_IGNORE_MAX_BAZEL_VERSION' not in os.environ):
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800503 print('Please downgrade your bazel installation to version %s or lower to '
Mihai Maruseace0963c42018-12-20 14:27:40 -0800504 'build TensorFlow! To downgrade: download the installer for the old '
505 'version (from https://github.com/bazelbuild/bazel/releases) then '
506 'run the installer.' % max_version)
Jason Zamanb41761c2018-10-14 11:28:53 +0800507 sys.exit(1)
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700508 return curr_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700509
510
511def set_cc_opt_flags(environ_cp):
512 """Set up architecture-dependent optimization flags.
513
514 Also append CC optimization flags to bazel.rc..
515
516 Args:
517 environ_cp: copy of the os.environ.
518 """
519 if is_ppc64le():
520 # gcc on ppc64le does not support -march, use mcpu instead
521 default_cc_opt_flags = '-mcpu=native'
A. Unique TensorFlower1bba94a2018-04-04 15:45:20 -0700522 elif is_windows():
523 default_cc_opt_flags = '/arch:AVX'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700524 else:
Justin Lebar9ef04f52018-10-10 18:52:45 -0700525 default_cc_opt_flags = '-march=native -Wno-sign-compare'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700526 question = ('Please specify optimization flags to use during compilation when'
527 ' bazel option "--config=opt" is specified [Default is %s]: '
528 ) % default_cc_opt_flags
529 cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS',
530 question, default_cc_opt_flags)
531 for opt in cc_opt_flags.split():
Michael Case00177422017-11-10 13:14:03 -0800532 write_to_bazelrc('build:opt --copt=%s' % opt)
533 # It should be safe on the same build host.
A. Unique TensorFlower1bba94a2018-04-04 15:45:20 -0700534 if not is_ppc64le() and not is_windows():
Shanqing Cai71445712018-03-12 19:33:52 -0700535 write_to_bazelrc('build:opt --host_copt=-march=native')
Michael Casebb3355d2017-11-09 08:46:31 -0800536 write_to_bazelrc('build:opt --define with_default_optimizations=true')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700537
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700538
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700539def set_tf_cuda_clang(environ_cp):
540 """set TF_CUDA_CLANG action_env.
541
542 Args:
543 environ_cp: copy of the os.environ.
544 """
545 question = 'Do you want to use clang as CUDA compiler?'
546 yes_reply = 'Clang will be used as CUDA compiler.'
547 no_reply = 'nvcc will be used as CUDA compiler.'
548 set_action_env_var(
549 environ_cp,
550 'TF_CUDA_CLANG',
551 None,
552 False,
553 question=question,
554 yes_reply=yes_reply,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700555 no_reply=no_reply,
556 bazel_config_name='cuda_clang')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700557
558
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800559def set_tf_download_clang(environ_cp):
560 """Set TF_DOWNLOAD_CLANG action_env."""
Ilya Biryukov9e651e42018-03-22 05:33:42 -0700561 question = 'Do you wish to download a fresh release of clang? (Experimental)'
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800562 yes_reply = 'Clang will be downloaded and used to compile tensorflow.'
563 no_reply = 'Clang will not be downloaded.'
564 set_action_env_var(
565 environ_cp,
566 'TF_DOWNLOAD_CLANG',
567 None,
568 False,
569 question=question,
570 yes_reply=yes_reply,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700571 no_reply=no_reply,
572 bazel_config_name='download_clang')
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800573
574
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700575def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var,
576 var_default):
577 """Get var_name either from env, or user or default.
578
579 If var_name has been set as environment variable, use the preset value, else
580 ask for user input. If no input is provided, the default is used.
581
582 Args:
583 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530584 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700585 ask_for_var: string for how to ask for user input.
586 var_default: default value string.
587
588 Returns:
589 string value for var_name
590 """
591 var = environ_cp.get(var_name)
592 if not var:
593 var = get_input(ask_for_var)
Michael Cased94271a2017-08-22 17:26:52 -0700594 print('\n')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700595 if not var:
596 var = var_default
597 return var
598
599
600def set_clang_cuda_compiler_path(environ_cp):
601 """Set CLANG_CUDA_COMPILER_PATH."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700602 default_clang_path = which('clang') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700603 ask_clang_path = ('Please specify which clang should be used as device and '
604 'host compiler. [Default is %s]: ') % default_clang_path
605
606 while True:
607 clang_cuda_compiler_path = get_from_env_or_user_or_default(
608 environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path,
609 default_clang_path)
610 if os.path.exists(clang_cuda_compiler_path):
611 break
612
613 # Reset and retry
614 print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path)
615 environ_cp['CLANG_CUDA_COMPILER_PATH'] = ''
616
617 # Set CLANG_CUDA_COMPILER_PATH
618 environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path
619 write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH',
620 clang_cuda_compiler_path)
621
622
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700623def prompt_loop_or_load_from_env(environ_cp,
624 var_name,
625 var_default,
626 ask_for_var,
627 check_success,
628 error_msg,
629 suppress_default_error=False,
Austin Anderson58b236b2019-10-22 10:36:10 -0700630 resolve_symlinks=False,
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700631 n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS):
Austin Anderson6afface2017-12-05 11:59:17 -0800632 """Loop over user prompts for an ENV param until receiving a valid response.
633
634 For the env param var_name, read from the environment or verify user input
635 until receiving valid input. When done, set var_name in the environ_cp to its
636 new value.
637
638 Args:
639 environ_cp: (Dict) copy of the os.environ.
640 var_name: (String) string for name of environment variable, e.g. "TF_MYVAR".
641 var_default: (String) default value string.
642 ask_for_var: (String) string for how to ask for user input.
643 check_success: (Function) function that takes one argument and returns a
644 boolean. Should return True if the value provided is considered valid. May
645 contain a complex error message if error_msg does not provide enough
646 information. In that case, set suppress_default_error to True.
647 error_msg: (String) String with one and only one '%s'. Formatted with each
648 invalid response upon check_success(input) failure.
649 suppress_default_error: (Bool) Suppress the above error message in favor of
650 one from the check_success function.
Austin Anderson58b236b2019-10-22 10:36:10 -0700651 resolve_symlinks: (Bool) Translate symbolic links into the real filepath.
Austin Anderson6afface2017-12-05 11:59:17 -0800652 n_ask_attempts: (Integer) Number of times to query for valid input before
653 raising an error and quitting.
654
655 Returns:
656 [String] The value of var_name after querying for input.
657
658 Raises:
659 UserInputError: if a query has been attempted n_ask_attempts times without
Frank Chenc4ef9272018-01-10 11:36:52 -0800660 success, assume that the user has made a scripting error, and will
661 continue to provide invalid input. Raise the error to avoid infinitely
662 looping.
Austin Anderson6afface2017-12-05 11:59:17 -0800663 """
664 default = environ_cp.get(var_name) or var_default
665 full_query = '%s [Default is %s]: ' % (
666 ask_for_var,
667 default,
668 )
669
670 for _ in range(n_ask_attempts):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700671 val = get_from_env_or_user_or_default(environ_cp, var_name, full_query,
Austin Anderson6afface2017-12-05 11:59:17 -0800672 default)
673 if check_success(val):
674 break
675 if not suppress_default_error:
676 print(error_msg % val)
677 environ_cp[var_name] = ''
678 else:
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700679 raise UserInputError('Invalid %s setting was provided %d times in a row. '
680 'Assuming to be a scripting mistake.' %
681 (var_name, n_ask_attempts))
Austin Anderson6afface2017-12-05 11:59:17 -0800682
Austin Anderson58b236b2019-10-22 10:36:10 -0700683 if resolve_symlinks and os.path.islink(val):
684 val = os.path.realpath(val)
Austin Anderson6afface2017-12-05 11:59:17 -0800685 environ_cp[var_name] = val
686 return val
687
688
689def create_android_ndk_rule(environ_cp):
690 """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule."""
691 if is_windows() or is_cygwin():
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700692 default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' %
693 environ_cp['APPDATA'])
Austin Anderson6afface2017-12-05 11:59:17 -0800694 elif is_macos():
695 default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME']
696 else:
697 default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME']
698
699 def valid_ndk_path(path):
700 return (os.path.exists(path) and
701 os.path.exists(os.path.join(path, 'source.properties')))
702
703 android_ndk_home_path = prompt_loop_or_load_from_env(
704 environ_cp,
705 var_name='ANDROID_NDK_HOME',
706 var_default=default_ndk_path,
707 ask_for_var='Please specify the home path of the Android NDK to use.',
708 check_success=valid_ndk_path,
709 error_msg=('The path %s or its child file "source.properties" '
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700710 'does not exist.'))
Michael Case51053502018-06-05 17:47:19 -0700711 write_action_env_to_bazelrc('ANDROID_NDK_HOME', android_ndk_home_path)
Jared Dukea0104b72019-04-04 12:23:58 -0700712 write_action_env_to_bazelrc(
713 'ANDROID_NDK_API_LEVEL',
714 get_ndk_api_level(environ_cp, android_ndk_home_path))
Austin Anderson6afface2017-12-05 11:59:17 -0800715
716
717def create_android_sdk_rule(environ_cp):
718 """Set Android variables and write Android SDK WORKSPACE rule."""
719 if is_windows() or is_cygwin():
720 default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA'])
721 elif is_macos():
Shashi Shekharc0ff0cc2018-07-17 09:00:24 -0700722 default_sdk_path = '%s/library/Android/Sdk' % environ_cp['HOME']
Austin Anderson6afface2017-12-05 11:59:17 -0800723 else:
724 default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME']
725
726 def valid_sdk_path(path):
727 return (os.path.exists(path) and
728 os.path.exists(os.path.join(path, 'platforms')) and
729 os.path.exists(os.path.join(path, 'build-tools')))
730
731 android_sdk_home_path = prompt_loop_or_load_from_env(
732 environ_cp,
733 var_name='ANDROID_SDK_HOME',
734 var_default=default_sdk_path,
735 ask_for_var='Please specify the home path of the Android SDK to use.',
736 check_success=valid_sdk_path,
737 error_msg=('Either %s does not exist, or it does not contain the '
738 'subdirectories "platforms" and "build-tools".'))
739
740 platforms = os.path.join(android_sdk_home_path, 'platforms')
741 api_levels = sorted(os.listdir(platforms))
742 api_levels = [x.replace('android-', '') for x in api_levels]
743
744 def valid_api_level(api_level):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700745 return os.path.exists(
746 os.path.join(android_sdk_home_path, 'platforms',
747 'android-' + api_level))
Austin Anderson6afface2017-12-05 11:59:17 -0800748
749 android_api_level = prompt_loop_or_load_from_env(
750 environ_cp,
751 var_name='ANDROID_API_LEVEL',
752 var_default=api_levels[-1],
753 ask_for_var=('Please specify the Android SDK API level to use. '
754 '[Available levels: %s]') % api_levels,
755 check_success=valid_api_level,
756 error_msg='Android-%s is not present in the SDK path.')
757
758 build_tools = os.path.join(android_sdk_home_path, 'build-tools')
759 versions = sorted(os.listdir(build_tools))
760
761 def valid_build_tools(version):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700762 return os.path.exists(
763 os.path.join(android_sdk_home_path, 'build-tools', version))
Austin Anderson6afface2017-12-05 11:59:17 -0800764
765 android_build_tools_version = prompt_loop_or_load_from_env(
766 environ_cp,
767 var_name='ANDROID_BUILD_TOOLS_VERSION',
768 var_default=versions[-1],
769 ask_for_var=('Please specify an Android build tools version to use. '
770 '[Available versions: %s]') % versions,
771 check_success=valid_build_tools,
772 error_msg=('The selected SDK does not have build-tools version %s '
773 'available.'))
774
Michael Case51053502018-06-05 17:47:19 -0700775 write_action_env_to_bazelrc('ANDROID_BUILD_TOOLS_VERSION',
776 android_build_tools_version)
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700777 write_action_env_to_bazelrc('ANDROID_SDK_API_LEVEL', android_api_level)
778 write_action_env_to_bazelrc('ANDROID_SDK_HOME', android_sdk_home_path)
Austin Anderson6afface2017-12-05 11:59:17 -0800779
780
Jared Dukea0104b72019-04-04 12:23:58 -0700781def get_ndk_api_level(environ_cp, android_ndk_home_path):
782 """Gets the appropriate NDK API level to use for the provided Android NDK path."""
783
784 # First check to see if we're using a blessed version of the NDK.
Austin Anderson6afface2017-12-05 11:59:17 -0800785 properties_path = '%s/source.properties' % android_ndk_home_path
786 if is_windows() or is_cygwin():
787 properties_path = cygpath(properties_path)
788 with open(properties_path, 'r') as f:
789 filedata = f.read()
790
791 revision = re.search(r'Pkg.Revision = (\d+)', filedata)
792 if revision:
Jared Dukea0104b72019-04-04 12:23:58 -0700793 ndk_version = revision.group(1)
Michael Case51053502018-06-05 17:47:19 -0700794 else:
795 raise Exception('Unable to parse NDK revision.')
Jared Dukea0104b72019-04-04 12:23:58 -0700796 if int(ndk_version) not in _SUPPORTED_ANDROID_NDK_VERSIONS:
797 print('WARNING: The NDK version in %s is %s, which is not '
798 'supported by Bazel (officially supported versions: %s). Please use '
799 'another version. Compiling Android targets may result in confusing '
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700800 'errors.\n' %
801 (android_ndk_home_path, ndk_version, _SUPPORTED_ANDROID_NDK_VERSIONS))
Jared Dukea0104b72019-04-04 12:23:58 -0700802
803 # Now grab the NDK API level to use. Note that this is different from the
804 # SDK API level, as the NDK API level is effectively the *min* target SDK
805 # version.
806 platforms = os.path.join(android_ndk_home_path, 'platforms')
807 api_levels = sorted(os.listdir(platforms))
808 api_levels = [
809 x.replace('android-', '') for x in api_levels if 'android-' in x
810 ]
811
812 def valid_api_level(api_level):
813 return os.path.exists(
814 os.path.join(android_ndk_home_path, 'platforms',
815 'android-' + api_level))
816
817 android_ndk_api_level = prompt_loop_or_load_from_env(
818 environ_cp,
819 var_name='ANDROID_NDK_API_LEVEL',
820 var_default='18', # 18 is required for GPU acceleration.
821 ask_for_var=('Please specify the (min) Android NDK API level to use. '
822 '[Available levels: %s]') % api_levels,
823 check_success=valid_api_level,
824 error_msg='Android-%s is not present in the NDK path.')
825
826 return android_ndk_api_level
Austin Anderson6afface2017-12-05 11:59:17 -0800827
828
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700829def set_gcc_host_compiler_path(environ_cp):
830 """Set GCC_HOST_COMPILER_PATH."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700831 default_gcc_host_compiler_path = which('gcc') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700832 cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH')
833
834 if os.path.islink(cuda_bin_symlink):
835 # os.readlink is only available in linux
Jonathan Hseu008910f2017-08-25 14:01:05 -0700836 default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700837
Austin Anderson6afface2017-12-05 11:59:17 -0800838 gcc_host_compiler_path = prompt_loop_or_load_from_env(
839 environ_cp,
840 var_name='GCC_HOST_COMPILER_PATH',
841 var_default=default_gcc_host_compiler_path,
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800842 ask_for_var='Please specify which gcc should be used by nvcc as the host compiler.',
Austin Anderson6afface2017-12-05 11:59:17 -0800843 check_success=os.path.exists,
Austin Anderson58b236b2019-10-22 10:36:10 -0700844 resolve_symlinks=True,
Austin Anderson6afface2017-12-05 11:59:17 -0800845 error_msg='Invalid gcc path. %s cannot be found.',
846 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700847
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700848 write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path)
849
850
Ankur Taly0e6f39d2018-02-16 18:22:55 -0800851def reformat_version_sequence(version_str, sequence_count):
852 """Reformat the version string to have the given number of sequences.
853
854 For example:
855 Given (7, 2) -> 7.0
856 (7.0.1, 2) -> 7.0
857 (5, 1) -> 5
858 (5.0.3.2, 1) -> 5
859
860 Args:
861 version_str: String, the version string.
862 sequence_count: int, an integer.
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700863
Ankur Taly0e6f39d2018-02-16 18:22:55 -0800864 Returns:
865 string, reformatted version string.
866 """
867 v = version_str.split('.')
868 if len(v) < sequence_count:
869 v = v + (['0'] * (sequence_count - len(v)))
870
871 return '.'.join(v[:sequence_count])
872
873
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700874def set_tf_cuda_paths(environ_cp):
875 """Set TF_CUDA_PATHS."""
876 ask_cuda_paths = (
877 'Please specify the comma-separated list of base paths to look for CUDA '
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700878 'libraries and headers. [Leave empty to use the default]: ')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700879 tf_cuda_paths = get_from_env_or_user_or_default(environ_cp, 'TF_CUDA_PATHS',
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700880 ask_cuda_paths, '')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700881 if tf_cuda_paths:
882 environ_cp['TF_CUDA_PATHS'] = tf_cuda_paths
883
884
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700885def set_tf_cuda_version(environ_cp):
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700886 """Set TF_CUDA_VERSION."""
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700887 ask_cuda_version = (
888 'Please specify the CUDA SDK version you want to use. '
889 '[Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700890 tf_cuda_version = get_from_env_or_user_or_default(environ_cp,
891 'TF_CUDA_VERSION',
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700892 ask_cuda_version,
893 _DEFAULT_CUDA_VERSION)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700894 environ_cp['TF_CUDA_VERSION'] = tf_cuda_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700895
896
Yifei Fengb1d8c592017-11-22 13:42:21 -0800897def set_tf_cudnn_version(environ_cp):
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700898 """Set TF_CUDNN_VERSION."""
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700899 ask_cudnn_version = (
900 'Please specify the cuDNN version you want to use. '
901 '[Leave empty to default to cuDNN %s]: ') % _DEFAULT_CUDNN_VERSION
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700902 tf_cudnn_version = get_from_env_or_user_or_default(environ_cp,
903 'TF_CUDNN_VERSION',
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700904 ask_cudnn_version,
905 _DEFAULT_CUDNN_VERSION)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700906 environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700907
908
Mingxing Tan1e7b0e42018-06-28 19:13:20 -0700909def is_cuda_compatible(lib, cuda_ver, cudnn_ver):
910 """Check compatibility between given library and cudnn/cudart libraries."""
911 ldd_bin = which('ldd') or '/usr/bin/ldd'
912 ldd_out = run_shell([ldd_bin, lib], True)
913 ldd_out = ldd_out.split(os.linesep)
914 cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$')
915 cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$')
916 cudnn = None
917 cudart = None
918 cudnn_ok = True # assume no cudnn dependency by default
919 cuda_ok = True # assume no cuda dependency by default
920 for line in ldd_out:
921 if 'libcudnn.so' in line:
922 cudnn = cudnn_pattern.search(line)
923 cudnn_ok = False
924 elif 'libcudart.so' in line:
925 cudart = cuda_pattern.search(line)
926 cuda_ok = False
927 if cudnn and len(cudnn.group(1)):
928 cudnn = convert_version_to_int(cudnn.group(1))
929 if cudart and len(cudart.group(1)):
930 cudart = convert_version_to_int(cudart.group(1))
931 if cudnn is not None:
932 cudnn_ok = (cudnn == cudnn_ver)
933 if cudart is not None:
934 cuda_ok = (cudart == cuda_ver)
935 return cudnn_ok and cuda_ok
936
937
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700938def set_tf_tensorrt_version(environ_cp):
939 """Set TF_TENSORRT_VERSION."""
Guangda Lai76f69382018-01-25 23:59:19 -0800940 if not is_linux():
941 raise ValueError('Currently TensorRT is only supported on Linux platform.')
942
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700943 if not int(environ_cp.get('TF_NEED_TENSORRT', False)):
Guangda Lai76f69382018-01-25 23:59:19 -0800944 return
945
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700946 ask_tensorrt_version = (
947 'Please specify the TensorRT version you want to use. '
Guangda Laifaa93ac2019-10-23 15:07:12 -0700948 '[Leave empty to default to TensorRT %s]: ') % _DEFAULT_TENSORRT_VERSION
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700949 tf_tensorrt_version = get_from_env_or_user_or_default(
950 environ_cp, 'TF_TENSORRT_VERSION', ask_tensorrt_version,
951 _DEFAULT_TENSORRT_VERSION)
Guangda Lai76f69382018-01-25 23:59:19 -0800952 environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version
Guangda Lai76f69382018-01-25 23:59:19 -0800953
954
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700955def set_tf_nccl_version(environ_cp):
956 """Set TF_NCCL_VERSION."""
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700957 if not is_linux():
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700958 raise ValueError('Currently NCCL is only supported on Linux platform.')
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700959
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700960 if 'TF_NCCL_VERSION' in environ_cp:
961 return
962
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700963 ask_nccl_version = (
A. Unique TensorFlower53faa312018-10-05 08:46:54 -0700964 'Please specify the locally installed NCCL version you want to use. '
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700965 '[Leave empty to use http://github.com/nvidia/nccl]: ')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700966 tf_nccl_version = get_from_env_or_user_or_default(environ_cp,
967 'TF_NCCL_VERSION',
968 ask_nccl_version, '')
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700969 environ_cp['TF_NCCL_VERSION'] = tf_nccl_version
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800970
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700971
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700972def get_native_cuda_compute_capabilities(environ_cp):
973 """Get native cuda compute capabilities.
974
975 Args:
976 environ_cp: copy of the os.environ.
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700977
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700978 Returns:
979 string of native cuda compute capabilities, separated by comma.
980 """
981 device_query_bin = os.path.join(
982 environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery')
Jonathan Hseu008910f2017-08-25 14:01:05 -0700983 if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK):
984 try:
985 output = run_shell(device_query_bin).split('\n')
986 pattern = re.compile('[0-9]*\\.[0-9]*')
987 output = [pattern.search(x) for x in output if 'Capability' in x]
988 output = ','.join(x.group() for x in output if x is not None)
989 except subprocess.CalledProcessError:
990 output = ''
991 else:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700992 output = ''
993 return output
994
995
996def set_tf_cuda_compute_capabilities(environ_cp):
997 """Set TF_CUDA_COMPUTE_CAPABILITIES."""
998 while True:
999 native_cuda_compute_capabilities = get_native_cuda_compute_capabilities(
1000 environ_cp)
1001 if not native_cuda_compute_capabilities:
1002 default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES
1003 else:
1004 default_cuda_compute_capabilities = native_cuda_compute_capabilities
1005
1006 ask_cuda_compute_capabilities = (
1007 'Please specify a list of comma-separated '
P Sudeepam52093562019-02-17 17:34:01 +05301008 'CUDA compute capabilities you want to '
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001009 'build with.\nYou can find the compute '
1010 'capability of your device at: '
1011 'https://developer.nvidia.com/cuda-gpus.\nPlease'
1012 ' note that each additional compute '
1013 'capability significantly increases your '
P Sudeepam52093562019-02-17 17:34:01 +05301014 'build time and binary size, and that '
1015 'TensorFlow only supports compute '
P Sudeepam765ceda2019-02-17 17:39:08 +05301016 'capabilities >= 3.5 [Default is: %s]: ' %
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001017 default_cuda_compute_capabilities)
1018 tf_cuda_compute_capabilities = get_from_env_or_user_or_default(
1019 environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES',
1020 ask_cuda_compute_capabilities, default_cuda_compute_capabilities)
1021 # Check whether all capabilities from the input is valid
1022 all_valid = True
Maciejd0f5bc12018-04-30 22:30:58 -05001023 # Remove all whitespace characters before splitting the string
Michael Case51053502018-06-05 17:47:19 -07001024 # that users may insert by accident, as this will result in error
Maciejd0f5bc12018-04-30 22:30:58 -05001025 tf_cuda_compute_capabilities = ''.join(tf_cuda_compute_capabilities.split())
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001026 for compute_capability in tf_cuda_compute_capabilities.split(','):
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001027 m = re.match('[0-9]+.[0-9]+', compute_capability)
1028 if not m:
Austin Anderson32202dc2019-02-19 10:46:27 -08001029 print('Invalid compute capability: %s' % compute_capability)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001030 all_valid = False
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001031 else:
P Sudeepam52093562019-02-17 17:34:01 +05301032 ver = float(m.group(0))
A. Unique TensorFlower8dc2d0e2019-03-12 01:41:05 -07001033 if ver < 3.0:
1034 print('ERROR: TensorFlow only supports CUDA compute capabilities 3.0 '
Austin Anderson32202dc2019-02-19 10:46:27 -08001035 'and higher. Please re-specify the list of compute '
1036 'capabilities excluding version %s.' % ver)
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001037 all_valid = False
A. Unique TensorFlower8dc2d0e2019-03-12 01:41:05 -07001038 if ver < 3.5:
1039 print('WARNING: XLA does not support CUDA compute capabilities '
1040 'lower than 3.5. Disable XLA when running on older GPUs.')
P Sudeepam765ceda2019-02-17 17:39:08 +05301041
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001042 if all_valid:
1043 break
1044
1045 # Reset and Retry
1046 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = ''
1047
1048 # Set TF_CUDA_COMPUTE_CAPABILITIES
1049 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities
1050 write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES',
1051 tf_cuda_compute_capabilities)
1052
1053
1054def set_other_cuda_vars(environ_cp):
1055 """Set other CUDA related variables."""
A. Unique TensorFlowerab391982018-07-11 04:52:49 -07001056 # If CUDA is enabled, always use GPU during build and test.
1057 if environ_cp.get('TF_CUDA_CLANG') == '1':
1058 write_to_bazelrc('build --config=cuda_clang')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001059 else:
A. Unique TensorFlowerab391982018-07-11 04:52:49 -07001060 write_to_bazelrc('build --config=cuda')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001061
1062
1063def set_host_cxx_compiler(environ_cp):
1064 """Set HOST_CXX_COMPILER."""
Jonathan Hseu008910f2017-08-25 14:01:05 -07001065 default_cxx_host_compiler = which('g++') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001066
Austin Anderson6afface2017-12-05 11:59:17 -08001067 host_cxx_compiler = prompt_loop_or_load_from_env(
1068 environ_cp,
1069 var_name='HOST_CXX_COMPILER',
1070 var_default=default_cxx_host_compiler,
1071 ask_for_var=('Please specify which C++ compiler should be used as the '
1072 'host C++ compiler.'),
1073 check_success=os.path.exists,
1074 error_msg='Invalid C++ compiler path. %s cannot be found.',
1075 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001076
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001077 write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler)
1078
1079
1080def set_host_c_compiler(environ_cp):
1081 """Set HOST_C_COMPILER."""
Jonathan Hseu008910f2017-08-25 14:01:05 -07001082 default_c_host_compiler = which('gcc') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001083
Austin Anderson6afface2017-12-05 11:59:17 -08001084 host_c_compiler = prompt_loop_or_load_from_env(
1085 environ_cp,
1086 var_name='HOST_C_COMPILER',
1087 var_default=default_c_host_compiler,
Shanqing Cai71445712018-03-12 19:33:52 -07001088 ask_for_var=('Please specify which C compiler should be used as the host '
Austin Anderson6afface2017-12-05 11:59:17 -08001089 'C compiler.'),
1090 check_success=os.path.exists,
1091 error_msg='Invalid C compiler path. %s cannot be found.',
1092 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001093
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001094 write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler)
1095
1096
1097def set_computecpp_toolkit_path(environ_cp):
1098 """Set COMPUTECPP_TOOLKIT_PATH."""
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001099
Austin Anderson6afface2017-12-05 11:59:17 -08001100 def toolkit_exists(toolkit_path):
1101 """Check if a computecpp toolkit path is valid."""
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001102 if is_linux():
1103 sycl_rt_lib_path = 'lib/libComputeCpp.so'
1104 else:
1105 sycl_rt_lib_path = ''
1106
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001107 sycl_rt_lib_path_full = os.path.join(toolkit_path, sycl_rt_lib_path)
Austin Anderson6afface2017-12-05 11:59:17 -08001108 exists = os.path.exists(sycl_rt_lib_path_full)
1109 if not exists:
1110 print('Invalid SYCL %s library path. %s cannot be found' %
1111 (_TF_OPENCL_VERSION, sycl_rt_lib_path_full))
1112 return exists
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001113
Austin Anderson6afface2017-12-05 11:59:17 -08001114 computecpp_toolkit_path = prompt_loop_or_load_from_env(
1115 environ_cp,
1116 var_name='COMPUTECPP_TOOLKIT_PATH',
1117 var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH,
1118 ask_for_var=(
1119 'Please specify the location where ComputeCpp for SYCL %s is '
1120 'installed.' % _TF_OPENCL_VERSION),
1121 check_success=toolkit_exists,
1122 error_msg='Invalid SYCL compiler path. %s cannot be found.',
1123 suppress_default_error=True)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001124
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001125 write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH',
1126 computecpp_toolkit_path)
1127
Michael Cased31531a2018-01-05 14:09:41 -08001128
Dandelion Man?90e42f32017-12-15 18:15:07 -08001129def set_trisycl_include_dir(environ_cp):
Michael Cased31531a2018-01-05 14:09:41 -08001130 """Set TRISYCL_INCLUDE_DIR."""
Frank Chenc4ef9272018-01-10 11:36:52 -08001131
Dandelion Man?90e42f32017-12-15 18:15:07 -08001132 ask_trisycl_include_dir = ('Please specify the location of the triSYCL '
1133 'include directory. (Use --config=sycl_trisycl '
1134 'when building with Bazel) '
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001135 '[Default is %s]: ') % (
1136 _DEFAULT_TRISYCL_INCLUDE_DIR)
Frank Chenc4ef9272018-01-10 11:36:52 -08001137
Dandelion Man?90e42f32017-12-15 18:15:07 -08001138 while True:
1139 trisycl_include_dir = get_from_env_or_user_or_default(
Michael Cased31531a2018-01-05 14:09:41 -08001140 environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir,
1141 _DEFAULT_TRISYCL_INCLUDE_DIR)
Dandelion Man?90e42f32017-12-15 18:15:07 -08001142 if os.path.exists(trisycl_include_dir):
1143 break
1144
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001145 print('Invalid triSYCL include directory, %s cannot be found' %
1146 (trisycl_include_dir))
Dandelion Man?90e42f32017-12-15 18:15:07 -08001147
1148 # Set TRISYCL_INCLUDE_DIR
1149 environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001150 write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', trisycl_include_dir)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001151
Yifei Fengb1d8c592017-11-22 13:42:21 -08001152
A. Unique TensorFlowere9797fd2019-03-22 11:02:40 -07001153def system_specific_test_config(env):
A. Unique TensorFlower7bd86372019-03-21 15:19:30 -07001154 """Add default build and test flags required for TF tests to bazelrc."""
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001155 write_to_bazelrc('test --flaky_test_attempts=3')
1156 write_to_bazelrc('test --test_size_filters=small,medium')
1157 write_to_bazelrc(
1158 'test --test_tag_filters=-benchmark-test,-no_oss,-oss_serial')
1159 write_to_bazelrc('test --build_tag_filters=-benchmark-test,-no_oss')
1160 if is_windows():
Guangda Laibcd701a2019-03-12 21:04:51 -07001161 if env.get('TF_NEED_CUDA', None) == '1':
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001162 write_to_bazelrc(
1163 'test --test_tag_filters=-no_windows,-no_windows_gpu,-no_gpu')
1164 write_to_bazelrc(
1165 'test --build_tag_filters=-no_windows,-no_windows_gpu,-no_gpu')
1166 else:
1167 write_to_bazelrc('test --test_tag_filters=-no_windows,-gpu')
1168 write_to_bazelrc('test --build_tag_filters=-no_windows,-gpu')
1169 elif is_macos():
1170 write_to_bazelrc('test --test_tag_filters=-gpu,-nomac,-no_mac')
1171 write_to_bazelrc('test --build_tag_filters=-gpu,-nomac,-no_mac')
1172 elif is_linux():
Guangda Laibcd701a2019-03-12 21:04:51 -07001173 if env.get('TF_NEED_CUDA', None) == '1':
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001174 write_to_bazelrc('test --test_tag_filters=-no_gpu')
1175 write_to_bazelrc('test --build_tag_filters=-no_gpu')
1176 write_to_bazelrc('test --test_env=LD_LIBRARY_PATH')
1177 else:
1178 write_to_bazelrc('test --test_tag_filters=-gpu')
1179 write_to_bazelrc('test --build_tag_filters=-gpu')
1180
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001181
Yifei Feng5198cb82018-08-17 13:53:06 -07001182def set_system_libs_flag(environ_cp):
1183 syslibs = environ_cp.get('TF_SYSTEM_LIBS', '')
TensorFlower Gardener61a87202018-10-01 12:25:39 -07001184 if syslibs:
Jason Zaman5fc39bd2018-09-16 01:38:55 +08001185 if ',' in syslibs:
1186 syslibs = ','.join(sorted(syslibs.split(',')))
1187 else:
1188 syslibs = ','.join(sorted(syslibs.split()))
Yifei Feng5198cb82018-08-17 13:53:06 -07001189 write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs)
1190
Jason Zaman5fc39bd2018-09-16 01:38:55 +08001191 if 'PREFIX' in environ_cp:
1192 write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX'])
1193 if 'LIBDIR' in environ_cp:
1194 write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR'])
1195 if 'INCLUDEDIR' in environ_cp:
1196 write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR'])
1197
Yifei Feng5198cb82018-08-17 13:53:06 -07001198
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001199def set_windows_build_flags(environ_cp):
1200 """Set Windows specific build options."""
1201 # The non-monolithic build is not supported yet
1202 write_to_bazelrc('build --config monolithic')
1203 # Suppress warning messages
1204 write_to_bazelrc('build --copt=-w --host_copt=-w')
Loo Rong Jie31f10d22019-02-02 10:03:20 +08001205 # Fix winsock2.h conflicts
TensorFlower Gardener345cccf2019-02-28 15:22:59 -08001206 write_to_bazelrc(
Justin Lebar7b57c5a2019-04-30 12:24:29 -07001207 'build --copt=-DWIN32_LEAN_AND_MEAN --host_copt=-DWIN32_LEAN_AND_MEAN '
1208 '--copt=-DNOGDI --host_copt=-DNOGDI')
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001209 # Output more verbose information when something goes wrong
1210 write_to_bazelrc('build --verbose_failures')
1211 # The host and target platforms are the same in Windows build. So we don't
1212 # have to distinct them. This avoids building the same targets twice.
1213 write_to_bazelrc('build --distinct_host_configuration=false')
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001214
1215 if get_var(
1216 environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline',
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001217 True, ('Would you like to override eigen strong inline for some C++ '
1218 'compilation to reduce the compilation time?'),
1219 'Eigen strong inline overridden.', 'Not overriding eigen strong inline, '
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001220 'some compilations could take more than 20 mins.'):
1221 # Due to a known MSVC compiler issue
1222 # https://github.com/tensorflow/tensorflow/issues/10521
1223 # Overriding eigen strong inline speeds up the compiling of
1224 # conv_grad_ops_3d.cc and conv_ops_3d.cc by 20 minutes,
1225 # but this also hurts the performance. Let users decide what they want.
1226 write_to_bazelrc('build --define=override_eigen_strong_inline=true')
Dandelion Man?90e42f32017-12-15 18:15:07 -08001227
A. Unique TensorFlower061c3592017-11-13 14:21:04 -08001228
Michael Cased31531a2018-01-05 14:09:41 -08001229def config_info_line(name, help_text):
1230 """Helper function to print formatted help text for Bazel config options."""
1231 print('\t--config=%-12s\t# %s' % (name, help_text))
1232
1233
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001234def configure_ios():
1235 """Configures TensorFlow for iOS builds.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001236
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001237 This function will only be executed if `is_macos()` is true.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001238 """
1239 if not is_macos():
1240 return
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001241 for filepath in APPLE_BAZEL_FILES:
1242 existing_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath + '.apple')
1243 renamed_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath)
1244 symlink_force(existing_filepath, renamed_filepath)
1245 for filepath in IOS_FILES:
1246 filename = os.path.basename(filepath)
1247 new_filepath = os.path.join(_TF_WORKSPACE_ROOT, filename)
1248 symlink_force(filepath, new_filepath)
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001249
1250
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001251def validate_cuda_config(environ_cp):
1252 """Run find_cuda_config.py and return cuda_toolkit_path, or None."""
1253
1254 def maybe_encode_env(env):
1255 """Encodes unicode in env to str on Windows python 2.x."""
1256 if not is_windows() or sys.version_info[0] != 2:
1257 return env
1258 for k, v in env.items():
1259 if isinstance(k, unicode):
1260 k = k.encode('ascii')
1261 if isinstance(v, unicode):
1262 v = v.encode('ascii')
1263 env[k] = v
1264 return env
1265
1266 cuda_libraries = ['cuda', 'cudnn']
1267 if is_linux():
A. Unique TensorFlower28fc9cc2019-05-01 14:17:54 -07001268 if int(environ_cp.get('TF_NEED_TENSORRT', False)):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001269 cuda_libraries.append('tensorrt')
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001270 if environ_cp.get('TF_NCCL_VERSION', None):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001271 cuda_libraries.append('nccl')
1272
1273 proc = subprocess.Popen(
1274 [environ_cp['PYTHON_BIN_PATH'], 'third_party/gpus/find_cuda_config.py'] +
1275 cuda_libraries,
1276 stdout=subprocess.PIPE,
1277 env=maybe_encode_env(environ_cp))
1278
1279 if proc.wait():
1280 # Errors from find_cuda_config.py were sent to stderr.
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -07001281 print('Asking for detailed CUDA configuration...\n')
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001282 return False
1283
1284 config = dict(
1285 tuple(line.decode('ascii').rstrip().split(': ')) for line in proc.stdout)
1286
1287 print('Found CUDA %s in:' % config['cuda_version'])
1288 print(' %s' % config['cuda_library_dir'])
1289 print(' %s' % config['cuda_include_dir'])
1290
1291 print('Found cuDNN %s in:' % config['cudnn_version'])
1292 print(' %s' % config['cudnn_library_dir'])
1293 print(' %s' % config['cudnn_include_dir'])
1294
1295 if 'tensorrt_version' in config:
1296 print('Found TensorRT %s in:' % config['tensorrt_version'])
1297 print(' %s' % config['tensorrt_library_dir'])
1298 print(' %s' % config['tensorrt_include_dir'])
1299
1300 if config.get('nccl_version', None):
1301 print('Found NCCL %s in:' % config['nccl_version'])
1302 print(' %s' % config['nccl_library_dir'])
1303 print(' %s' % config['nccl_include_dir'])
1304
1305 print('\n')
1306
1307 environ_cp['CUDA_TOOLKIT_PATH'] = config['cuda_toolkit_path']
1308 return True
1309
1310
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001311def main():
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001312 global _TF_WORKSPACE_ROOT
1313 global _TF_BAZELRC
A. Unique TensorFlowered297342019-03-15 11:25:28 -07001314 global _TF_CURRENT_BAZEL_VERSION
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001315
Shanqing Cai71445712018-03-12 19:33:52 -07001316 parser = argparse.ArgumentParser()
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001317 parser.add_argument(
1318 '--workspace',
1319 type=str,
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001320 default=os.path.abspath(os.path.dirname(__file__)),
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001321 help='The absolute path to your active Bazel workspace.')
Shanqing Cai71445712018-03-12 19:33:52 -07001322 args = parser.parse_args()
1323
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001324 _TF_WORKSPACE_ROOT = args.workspace
1325 _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME)
1326
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001327 # Make a copy of os.environ to be clear when functions and getting and setting
1328 # environment variables.
1329 environ_cp = dict(os.environ)
1330
Mark Daoust44000ad2019-06-18 09:26:26 -07001331 current_bazel_version = check_bazel_version(_TF_MIN_BAZEL_VERSION,
1332 _TF_MAX_BAZEL_VERSION)
A. Unique TensorFlowered297342019-03-15 11:25:28 -07001333 _TF_CURRENT_BAZEL_VERSION = convert_version_to_int(current_bazel_version)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001334
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001335 reset_tf_configure_bazelrc()
Yun Peng03e63a22018-11-07 11:18:53 +01001336
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001337 cleanup_makefile()
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -07001338 setup_python(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001339
1340 if is_windows():
Yifei Fengb1d8c592017-11-22 13:42:21 -08001341 environ_cp['TF_NEED_OPENCL_SYCL'] = '0'
1342 environ_cp['TF_NEED_COMPUTECPP'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001343 environ_cp['TF_NEED_OPENCL'] = '0'
1344 environ_cp['TF_CUDA_CLANG'] = '0'
Guangda Lai76f69382018-01-25 23:59:19 -08001345 environ_cp['TF_NEED_TENSORRT'] = '0'
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001346 # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on
1347 # Windows.
1348 environ_cp['TF_DOWNLOAD_CLANG'] = '0'
A. Unique TensorFlower6e97fb32018-07-16 14:07:29 -07001349 environ_cp['TF_NEED_MPI'] = '0'
1350 environ_cp['TF_SET_ANDROID_WORKSPACE'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001351
1352 if is_macos():
Guangda Lai76f69382018-01-25 23:59:19 -08001353 environ_cp['TF_NEED_TENSORRT'] = '0'
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001354 else:
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001355 environ_cp['TF_CONFIGURE_IOS'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001356
Jon Triebenbach6896a742018-06-27 13:29:53 -05001357 # The numpy package on ppc64le uses OpenBLAS which has multi-threading
1358 # issues that lead to incorrect answers. Set OMP_NUM_THREADS=1 at
1359 # runtime to allow the Tensorflow testcases which compare numpy
1360 # results to Tensorflow results to succeed.
1361 if is_ppc64le():
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001362 write_action_env_to_bazelrc('OMP_NUM_THREADS', 1)
Jon Triebenbach6896a742018-06-27 13:29:53 -05001363
Sanjoy Dasf3fab7c2019-06-20 11:19:00 -07001364 xla_enabled_by_default = is_linux() or is_macos()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001365 set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
Grzegorz Pawelczakec82efd2018-10-09 15:03:46 +01001366 xla_enabled_by_default, 'xla')
Gunhan Gulsoy3da0dff2018-09-26 11:55:50 -07001367
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001368 set_action_env_var(
1369 environ_cp,
1370 'TF_NEED_OPENCL_SYCL',
1371 'OpenCL SYCL',
1372 False,
1373 bazel_config_name='sycl')
Yifei Fengb1d8c592017-11-22 13:42:21 -08001374 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1':
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001375 set_host_cxx_compiler(environ_cp)
1376 set_host_c_compiler(environ_cp)
Yifei Fengb1d8c592017-11-22 13:42:21 -08001377 set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True)
1378 if environ_cp.get('TF_NEED_COMPUTECPP') == '1':
1379 set_computecpp_toolkit_path(environ_cp)
1380 else:
1381 set_trisycl_include_dir(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001382
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001383 set_action_env_var(
1384 environ_cp, 'TF_NEED_ROCM', 'ROCm', False, bazel_config_name='rocm')
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001385 if (environ_cp.get('TF_NEED_ROCM') == '1' and
TensorFlower Gardener62e60162018-09-27 10:22:55 -07001386 'LD_LIBRARY_PATH' in environ_cp and
1387 environ_cp.get('LD_LIBRARY_PATH') != '1'):
1388 write_action_env_to_bazelrc('LD_LIBRARY_PATH',
1389 environ_cp.get('LD_LIBRARY_PATH'))
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001390
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001391 environ_cp['TF_NEED_CUDA'] = str(
1392 int(get_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False)))
A. Unique TensorFlower24cbb2a2017-09-08 07:45:44 -07001393 if (environ_cp.get('TF_NEED_CUDA') == '1' and
1394 'TF_CUDA_CONFIG_REPO' not in environ_cp):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001395
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001396 set_action_env_var(
1397 environ_cp,
1398 'TF_NEED_TENSORRT',
1399 'TensorRT',
1400 False,
1401 bazel_config_name='tensorrt')
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001402
1403 environ_save = dict(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001404 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001405
1406 if validate_cuda_config(environ_cp):
1407 cuda_env_names = [
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001408 'TF_CUDA_VERSION',
1409 'TF_CUBLAS_VERSION',
1410 'TF_CUDNN_VERSION',
1411 'TF_TENSORRT_VERSION',
1412 'TF_NCCL_VERSION',
1413 'TF_CUDA_PATHS',
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001414 # Items below are for backwards compatibility when not using
1415 # TF_CUDA_PATHS.
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001416 'CUDA_TOOLKIT_PATH',
1417 'CUDNN_INSTALL_PATH',
1418 'NCCL_INSTALL_PATH',
1419 'NCCL_HDR_PATH',
1420 'TENSORRT_INSTALL_PATH'
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001421 ]
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001422 # Note: set_action_env_var above already writes to bazelrc.
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001423 for name in cuda_env_names:
1424 if name in environ_cp:
1425 write_action_env_to_bazelrc(name, environ_cp[name])
1426 break
1427
1428 # Restore settings changed below if CUDA config could not be validated.
1429 environ_cp = dict(environ_save)
1430
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001431 set_tf_cuda_version(environ_cp)
1432 set_tf_cudnn_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001433 if is_linux():
1434 set_tf_tensorrt_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001435 set_tf_nccl_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001436
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001437 set_tf_cuda_paths(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001438
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001439 else:
1440 raise UserInputError(
1441 'Invalid CUDA setting were provided %d '
1442 'times in a row. Assuming to be a scripting mistake.' %
1443 _DEFAULT_PROMPT_ASK_ATTEMPTS)
1444
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001445 set_tf_cuda_compute_capabilities(environ_cp)
Ankur Taly0e6f39d2018-02-16 18:22:55 -08001446 if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get(
1447 'LD_LIBRARY_PATH') != '1':
1448 write_action_env_to_bazelrc('LD_LIBRARY_PATH',
1449 environ_cp.get('LD_LIBRARY_PATH'))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001450
1451 set_tf_cuda_clang(environ_cp)
1452 if environ_cp.get('TF_CUDA_CLANG') == '1':
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001453 # Ask whether we should download the clang toolchain.
1454 set_tf_download_clang(environ_cp)
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -08001455 if environ_cp.get('TF_DOWNLOAD_CLANG') != '1':
1456 # Set up which clang we should use as the cuda / host compiler.
1457 set_clang_cuda_compiler_path(environ_cp)
Ilya Biryukov1c3d02e2018-09-04 03:09:52 -07001458 else:
1459 # Use downloaded LLD for linking.
1460 write_to_bazelrc('build:cuda_clang --config=download_clang_use_lld')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001461 else:
1462 # Set up which gcc nvcc should use as the host compiler
1463 # No need to set this on Windows
1464 if not is_windows():
1465 set_gcc_host_compiler_path(environ_cp)
1466 set_other_cuda_vars(environ_cp)
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001467 else:
1468 # CUDA not required. Ask whether we should download the clang toolchain and
1469 # use it for the CPU build.
1470 set_tf_download_clang(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001471
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001472 # SYCL / ROCm / CUDA are mutually exclusive.
1473 # At most 1 GPU platform can be configured.
1474 gpu_platform_count = 0
1475 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1':
1476 gpu_platform_count += 1
1477 if environ_cp.get('TF_NEED_ROCM') == '1':
1478 gpu_platform_count += 1
1479 if environ_cp.get('TF_NEED_CUDA') == '1':
1480 gpu_platform_count += 1
1481 if gpu_platform_count >= 2:
1482 raise UserInputError('SYCL / CUDA / ROCm are mututally exclusive. '
1483 'At most 1 GPU platform can be configured.')
1484
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001485 set_cc_opt_flags(environ_cp)
Yifei Feng5198cb82018-08-17 13:53:06 -07001486 set_system_libs_flag(environ_cp)
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001487 if is_windows():
1488 set_windows_build_flags(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001489
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001490 if get_var(environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', False,
1491 ('Would you like to interactively configure ./WORKSPACE for '
1492 'Android builds?'), 'Searching for NDK and SDK installations.',
1493 'Not configuring the WORKSPACE for Android builds.'):
Michael Case51053502018-06-05 17:47:19 -07001494 create_android_ndk_rule(environ_cp)
1495 create_android_sdk_rule(environ_cp)
Austin Anderson6afface2017-12-05 11:59:17 -08001496
A. Unique TensorFlowere9797fd2019-03-22 11:02:40 -07001497 system_specific_test_config(os.environ)
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001498
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -07001499 set_action_env_var(environ_cp, 'TF_CONFIGURE_IOS', 'iOS', False)
1500 if environ_cp.get('TF_CONFIGURE_IOS') == '1':
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001501 configure_ios()
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001502
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001503 print('Preconfigured Bazel build configs. You can use any of the below by '
1504 'adding "--config=<>" to your build command. See .bazelrc for more '
1505 'details.')
1506 config_info_line('mkl', 'Build with MKL support.')
1507 config_info_line('monolithic', 'Config for mostly static monolithic build.')
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001508 config_info_line('ngraph', 'Build with Intel nGraph support.')
A. Unique TensorFlowera6bf9c82019-02-26 10:08:35 -08001509 config_info_line('numa', 'Build with NUMA support.')
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -08001510 config_info_line(
1511 'dynamic_kernels',
1512 '(Experimental) Build kernels into separate shared objects.')
Anna Raedf6742019-06-12 11:30:51 -07001513 config_info_line('v2', 'Build TensorFlow 2.x instead of 1.x.')
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001514
1515 print('Preconfigured Bazel build configs to DISABLE default on features:')
1516 config_info_line('noaws', 'Disable AWS S3 filesystem support.')
1517 config_info_line('nogcp', 'Disable GCP support.')
1518 config_info_line('nohdfs', 'Disable HDFS support.')
Gunhan Gulsoyeea81682018-11-26 16:51:23 -08001519 config_info_line('nonccl', 'Disable NVIDIA NCCL support.')
Austin Anderson6afface2017-12-05 11:59:17 -08001520
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001521
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001522if __name__ == '__main__':
1523 main()