blob: 4cb68924db4d69d6c543381fc69e1e897d83af4a [file] [log] [blame]
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""configure script to get build parameters from user."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
Shanqing Cai71445712018-03-12 19:33:52 -070021import argparse
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070022import errno
23import os
24import platform
25import re
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070026import subprocess
27import sys
28
Andrew Sellec9885ea2017-11-06 09:37:03 -080029# pylint: disable=g-import-not-at-top
Jonathan Hseu008910f2017-08-25 14:01:05 -070030try:
31 from shutil import which
32except ImportError:
33 from distutils.spawn import find_executable as which
Andrew Sellec9885ea2017-11-06 09:37:03 -080034# pylint: enable=g-import-not-at-top
Jonathan Hseu008910f2017-08-25 14:01:05 -070035
Christian Sigg8dff6b92019-12-09 04:05:46 -080036_DEFAULT_CUDA_VERSION = '10'
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -070037_DEFAULT_CUDNN_VERSION = '7'
Guangda Laifaa93ac2019-10-23 15:07:12 -070038_DEFAULT_TENSORRT_VERSION = '6'
Smit Hinsufe7d1d92018-07-14 13:16:58 -070039_DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0'
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -070040
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070041_TF_OPENCL_VERSION = '1.2'
42_DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp'
Yifei Fengb1d8c592017-11-22 13:42:21 -080043_DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include'
A. Unique TensorFlower82820ef2018-11-12 13:22:13 -080044_SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15, 16, 17, 18]
Austin Anderson6afface2017-12-05 11:59:17 -080045
46_DEFAULT_PROMPT_ASK_ATTEMPTS = 10
47
Shanqing Cai71445712018-03-12 19:33:52 -070048_TF_BAZELRC_FILENAME = '.tf_configure.bazelrc'
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -070049_TF_WORKSPACE_ROOT = ''
50_TF_BAZELRC = ''
A. Unique TensorFlowered297342019-03-15 11:25:28 -070051_TF_CURRENT_BAZEL_VERSION = None
Brian Zhaofeb0c7b2020-01-08 17:30:14 -080052_TF_MIN_BAZEL_VERSION = '1.2.1'
53_TF_MAX_BAZEL_VERSION = '1.2.1'
Shanqing Cai71445712018-03-12 19:33:52 -070054
Jason Furmanek7c234152018-09-26 04:44:12 +000055NCCL_LIB_PATHS = [
TensorFlower Gardenerea5c5292018-10-01 17:28:26 -070056 'lib64/', 'lib/powerpc64le-linux-gnu/', 'lib/x86_64-linux-gnu/', ''
Jason Furmanek7c234152018-09-26 04:44:12 +000057]
Austin Anderson6afface2017-12-05 11:59:17 -080058
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -070059# List of files to configure when building Bazel on Apple platforms.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -080060APPLE_BAZEL_FILES = [
A. Unique TensorFlower6a059c02019-04-05 14:28:27 -070061 'tensorflow/lite/experimental/ios/BUILD',
A. Unique TensorFlower93e70732019-02-14 16:45:32 -080062 'tensorflow/lite/experimental/objc/BUILD',
YoungSeok Yoon7089aaa2019-11-11 21:36:17 -080063 'tensorflow/lite/experimental/swift/BUILD',
64 'tensorflow/lite/tools/benchmark/experimental/ios/BUILD'
A. Unique TensorFlower93e70732019-02-14 16:45:32 -080065]
66
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -070067# List of files to move when building for iOS.
68IOS_FILES = [
69 'tensorflow/lite/experimental/objc/TensorFlowLiteObjC.podspec',
70 'tensorflow/lite/experimental/swift/TensorFlowLiteSwift.podspec',
71]
72
Austin Anderson6afface2017-12-05 11:59:17 -080073
74class UserInputError(Exception):
75 pass
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070076
77
78def is_windows():
79 return platform.system() == 'Windows'
80
81
82def is_linux():
83 return platform.system() == 'Linux'
84
85
86def is_macos():
87 return platform.system() == 'Darwin'
88
89
90def is_ppc64le():
91 return platform.machine() == 'ppc64le'
92
93
Jonathan Hseu008910f2017-08-25 14:01:05 -070094def is_cygwin():
95 return platform.system().startswith('CYGWIN_NT')
96
97
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070098def get_input(question):
99 try:
100 try:
101 answer = raw_input(question)
102 except NameError:
103 answer = input(question) # pylint: disable=bad-builtin
104 except EOFError:
105 answer = ''
106 return answer
107
108
109def symlink_force(target, link_name):
110 """Force symlink, equivalent of 'ln -sf'.
111
112 Args:
113 target: items to link to.
114 link_name: name of the link.
115 """
116 try:
117 os.symlink(target, link_name)
118 except OSError as e:
119 if e.errno == errno.EEXIST:
120 os.remove(link_name)
121 os.symlink(target, link_name)
122 else:
123 raise e
124
125
126def sed_in_place(filename, old, new):
127 """Replace old string with new string in file.
128
129 Args:
130 filename: string for filename.
131 old: string to replace.
132 new: new string to replace to.
133 """
134 with open(filename, 'r') as f:
135 filedata = f.read()
136 newdata = filedata.replace(old, new)
137 with open(filename, 'w') as f:
138 f.write(newdata)
139
140
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700141def write_to_bazelrc(line):
142 with open(_TF_BAZELRC, 'a') as f:
143 f.write(line + '\n')
144
145
146def write_action_env_to_bazelrc(var_name, var):
147 write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var)))
148
149
Wallyss Lima72371042019-10-03 12:55:30 -0300150def run_shell(cmd, allow_non_zero=False, stderr=None):
151 if stderr is None:
152 stderr = sys.stdout
Jonathan Hseu008910f2017-08-25 14:01:05 -0700153 if allow_non_zero:
154 try:
Wallyss Lima72371042019-10-03 12:55:30 -0300155 output = subprocess.check_output(cmd, stderr=stderr)
Jonathan Hseu008910f2017-08-25 14:01:05 -0700156 except subprocess.CalledProcessError as e:
157 output = e.output
158 else:
Wallyss Lima72371042019-10-03 12:55:30 -0300159 output = subprocess.check_output(cmd, stderr=stderr)
Jonathan Hseu008910f2017-08-25 14:01:05 -0700160 return output.decode('UTF-8').strip()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700161
162
163def cygpath(path):
164 """Convert path from posix to windows."""
Martin Wicked57572e2017-09-02 19:21:45 -0700165 return os.path.abspath(path).replace('\\', '/')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700166
167
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700168def get_python_path(environ_cp, python_bin_path):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700169 """Get the python site package paths."""
170 python_paths = []
171 if environ_cp.get('PYTHONPATH'):
172 python_paths = environ_cp.get('PYTHONPATH').split(':')
173 try:
Wallyss Lima484304e2019-10-03 12:58:20 -0300174 stderr = open(os.devnull, 'wb')
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700175 library_paths = run_shell([
176 python_bin_path, '-c',
177 'import site; print("\\n".join(site.getsitepackages()))'
TensorFlower Gardener839a4362019-12-30 02:23:29 -0800178 ],
179 stderr=stderr).split('\n')
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700180 except subprocess.CalledProcessError:
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700181 library_paths = [
182 run_shell([
183 python_bin_path, '-c',
184 'from distutils.sysconfig import get_python_lib;'
185 'print(get_python_lib())'
186 ])
187 ]
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700188
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700189 all_paths = set(python_paths + library_paths)
190
191 paths = []
192 for path in all_paths:
193 if os.path.isdir(path):
194 paths.append(path)
195 return paths
196
197
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700198def get_python_major_version(python_bin_path):
199 """Get the python major version."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700200 return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])'])
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700201
202
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -0700203def setup_python(environ_cp):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700204 """Setup python related env variables."""
205 # Get PYTHON_BIN_PATH, default is the current running python.
206 default_python_bin_path = sys.executable
207 ask_python_bin_path = ('Please specify the location of python. [Default is '
208 '%s]: ') % default_python_bin_path
209 while True:
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700210 python_bin_path = get_from_env_or_user_or_default(environ_cp,
211 'PYTHON_BIN_PATH',
212 ask_python_bin_path,
213 default_python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700214 # Check if the path is valid
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700215 if os.path.isfile(python_bin_path) and os.access(python_bin_path, os.X_OK):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700216 break
217 elif not os.path.exists(python_bin_path):
218 print('Invalid python path: %s cannot be found.' % python_bin_path)
219 else:
220 print('%s is not executable. Is it the python binary?' % python_bin_path)
221 environ_cp['PYTHON_BIN_PATH'] = ''
222
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700223 # Convert python path to Windows style before checking lib and version
Martin Wicked57572e2017-09-02 19:21:45 -0700224 if is_windows() or is_cygwin():
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700225 python_bin_path = cygpath(python_bin_path)
226
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700227 # Get PYTHON_LIB_PATH
228 python_lib_path = environ_cp.get('PYTHON_LIB_PATH')
229 if not python_lib_path:
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700230 python_lib_paths = get_python_path(environ_cp, python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700231 if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1':
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700232 python_lib_path = python_lib_paths[0]
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700233 else:
Jonathan Hseu008910f2017-08-25 14:01:05 -0700234 print('Found possible Python library paths:\n %s' %
235 '\n '.join(python_lib_paths))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700236 default_python_lib_path = python_lib_paths[0]
237 python_lib_path = get_input(
Jonathan Hseu008910f2017-08-25 14:01:05 -0700238 'Please input the desired Python library path to use. '
239 'Default is [%s]\n' % python_lib_paths[0])
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700240 if not python_lib_path:
241 python_lib_path = default_python_lib_path
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700242 environ_cp['PYTHON_LIB_PATH'] = python_lib_path
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700243
A. Unique TensorFlowercfccbdb2019-06-24 11:01:58 -0700244 python_major_version = get_python_major_version(python_bin_path)
245 if python_major_version == '2':
246 write_to_bazelrc('build --host_force_python=PY2')
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700247
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700248 # Convert python path to Windows style before writing into bazel.rc
Martin Wicked57572e2017-09-02 19:21:45 -0700249 if is_windows() or is_cygwin():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700250 python_lib_path = cygpath(python_lib_path)
251
252 # Set-up env variables used by python_configure.bzl
253 write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path)
254 write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path)
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -0700255 write_to_bazelrc('build --python_path=\"%s"' % python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700256 environ_cp['PYTHON_BIN_PATH'] = python_bin_path
257
William D. Ironsdcc76a52018-11-20 10:35:18 -0600258 # If choosen python_lib_path is from a path specified in the PYTHONPATH
259 # variable, need to tell bazel to include PYTHONPATH
260 if environ_cp.get('PYTHONPATH'):
261 python_paths = environ_cp.get('PYTHONPATH').split(':')
262 if python_lib_path in python_paths:
TensorFlower Gardener968cd182018-11-28 11:33:16 -0800263 write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH'))
William D. Ironsdcc76a52018-11-20 10:35:18 -0600264
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700265 # Write tools/python_bin_path.sh
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700266 with open(
267 os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'),
268 'w') as f:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700269 f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path)
270
271
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -0700272def reset_tf_configure_bazelrc():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700273 """Reset file that contains customized config settings."""
274 open(_TF_BAZELRC, 'w').close()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700275
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800276
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700277def cleanup_makefile():
278 """Delete any leftover BUILD files from the Makefile build.
279
280 These files could interfere with Bazel parsing.
281 """
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700282 makefile_download_dir = os.path.join(_TF_WORKSPACE_ROOT, 'tensorflow',
283 'contrib', 'makefile', 'downloads')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700284 if os.path.isdir(makefile_download_dir):
285 for root, _, filenames in os.walk(makefile_download_dir):
286 for f in filenames:
287 if f.endswith('BUILD'):
288 os.remove(os.path.join(root, f))
289
290
291def get_var(environ_cp,
292 var_name,
293 query_item,
294 enabled_by_default,
295 question=None,
296 yes_reply=None,
297 no_reply=None):
298 """Get boolean input from user.
299
300 If var_name is not set in env, ask user to enable query_item or not. If the
301 response is empty, use the default.
302
303 Args:
304 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530305 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
306 query_item: string for feature related to the variable, e.g. "CUDA for
307 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700308 enabled_by_default: boolean for default behavior.
309 question: optional string for how to ask for user input.
Michael Cased90054e2018-02-07 14:36:00 -0800310 yes_reply: optional string for reply when feature is enabled.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700311 no_reply: optional string for reply when feature is disabled.
312
313 Returns:
314 boolean value of the variable.
Frank Chenc4ef9272018-01-10 11:36:52 -0800315
316 Raises:
317 UserInputError: if an environment variable is set, but it cannot be
318 interpreted as a boolean indicator, assume that the user has made a
319 scripting error, and will continue to provide invalid input.
320 Raise the error to avoid infinitely looping.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700321 """
322 if not question:
323 question = 'Do you wish to build TensorFlow with %s support?' % query_item
324 if not yes_reply:
325 yes_reply = '%s support will be enabled for TensorFlow.' % query_item
326 if not no_reply:
327 no_reply = 'No %s' % yes_reply
328
329 yes_reply += '\n'
330 no_reply += '\n'
331
332 if enabled_by_default:
333 question += ' [Y/n]: '
334 else:
335 question += ' [y/N]: '
336
337 var = environ_cp.get(var_name)
Frank Chenc4ef9272018-01-10 11:36:52 -0800338 if var is not None:
339 var_content = var.strip().lower()
340 true_strings = ('1', 't', 'true', 'y', 'yes')
341 false_strings = ('0', 'f', 'false', 'n', 'no')
342 if var_content in true_strings:
343 var = True
344 elif var_content in false_strings:
345 var = False
346 else:
347 raise UserInputError(
348 'Environment variable %s must be set as a boolean indicator.\n'
349 'The following are accepted as TRUE : %s.\n'
350 'The following are accepted as FALSE: %s.\n'
A. Unique TensorFlowered297342019-03-15 11:25:28 -0700351 'Current value is %s.' %
352 (var_name, ', '.join(true_strings), ', '.join(false_strings), var))
Frank Chenc4ef9272018-01-10 11:36:52 -0800353
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700354 while var is None:
355 user_input_origin = get_input(question)
356 user_input = user_input_origin.strip().lower()
357 if user_input == 'y':
358 print(yes_reply)
359 var = True
360 elif user_input == 'n':
361 print(no_reply)
362 var = False
363 elif not user_input:
364 if enabled_by_default:
365 print(yes_reply)
366 var = True
367 else:
368 print(no_reply)
369 var = False
370 else:
371 print('Invalid selection: %s' % user_input_origin)
372 return var
373
374
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700375def set_build_var(environ_cp,
376 var_name,
377 query_item,
378 option_name,
379 enabled_by_default,
380 bazel_config_name=None):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700381 """Set if query_item will be enabled for the build.
382
383 Ask user if query_item will be enabled. Default is used if no input is given.
384 Set subprocess environment variable and write to .bazelrc if enabled.
385
386 Args:
387 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530388 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
389 query_item: string for feature related to the variable, e.g. "CUDA for
390 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700391 option_name: string for option to define in .bazelrc.
392 enabled_by_default: boolean for default behavior.
Michael Case98850a52017-09-14 13:35:57 -0700393 bazel_config_name: Name for Bazel --config argument to enable build feature.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700394 """
395
396 var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default)))
397 environ_cp[var_name] = var
398 if var == '1':
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700399 write_to_bazelrc('build:%s --define %s=true' %
400 (bazel_config_name, option_name))
Yifei Fengec451f52018-10-05 12:53:50 -0700401 write_to_bazelrc('build --config=%s' % bazel_config_name)
Michael Case98850a52017-09-14 13:35:57 -0700402 elif bazel_config_name is not None:
403 # TODO(mikecase): Migrate all users of configure.py to use --config Bazel
404 # options and not to set build configs through environment variables.
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700405 write_to_bazelrc('build:%s --define %s=true' %
406 (bazel_config_name, option_name))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700407
408
409def set_action_env_var(environ_cp,
410 var_name,
411 query_item,
412 enabled_by_default,
413 question=None,
414 yes_reply=None,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700415 no_reply=None,
416 bazel_config_name=None):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700417 """Set boolean action_env variable.
418
419 Ask user if query_item will be enabled. Default is used if no input is given.
420 Set environment variable and write to .bazelrc.
421
422 Args:
423 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530424 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
425 query_item: string for feature related to the variable, e.g. "CUDA for
426 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700427 enabled_by_default: boolean for default behavior.
428 question: optional string for how to ask for user input.
Michael Cased90054e2018-02-07 14:36:00 -0800429 yes_reply: optional string for reply when feature is enabled.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700430 no_reply: optional string for reply when feature is disabled.
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700431 bazel_config_name: adding config to .bazelrc instead of action_env.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700432 """
433 var = int(
434 get_var(environ_cp, var_name, query_item, enabled_by_default, question,
435 yes_reply, no_reply))
436
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700437 if not bazel_config_name:
438 write_action_env_to_bazelrc(var_name, var)
439 elif var:
440 write_to_bazelrc('build --config=%s' % bazel_config_name)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700441 environ_cp[var_name] = str(var)
442
443
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700444def convert_version_to_int(version):
445 """Convert a version number to a integer that can be used to compare.
446
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700447 Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The
448 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored.
449
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700450 Args:
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700451 version: a version to be converted
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700452
453 Returns:
454 An integer if converted successfully, otherwise return None.
455 """
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700456 version = version.split('-')[0]
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700457 version_segments = version.split('.')
Austin Anderson87ea41d2019-04-04 10:03:50 -0700458 # Treat "0.24" as "0.24.0"
459 if len(version_segments) == 2:
460 version_segments.append('0')
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700461 for seg in version_segments:
462 if not seg.isdigit():
463 return None
464
465 version_str = ''.join(['%03d' % int(seg) for seg in version_segments])
466 return int(version_str)
467
468
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800469def check_bazel_version(min_version, max_version):
470 """Check installed bazel version is between min_version and max_version.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700471
472 Args:
Mihai Maruseac4db860f2019-04-19 08:52:10 -0700473 min_version: string for minimum bazel version (must exist!).
474 max_version: string for maximum bazel version (must exist!).
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700475
476 Returns:
477 The bazel version detected.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700478 """
Jonathan Hseu008910f2017-08-25 14:01:05 -0700479 if which('bazel') is None:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700480 print('Cannot find bazel. Please install bazel.')
481 sys.exit(0)
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700482 curr_version = run_shell(
483 ['bazel', '--batch', '--bazelrc=/dev/null', 'version'])
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700484
485 for line in curr_version.split('\n'):
486 if 'Build label: ' in line:
487 curr_version = line.split('Build label: ')[1]
488 break
489
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700490 min_version_int = convert_version_to_int(min_version)
491 curr_version_int = convert_version_to_int(curr_version)
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800492 max_version_int = convert_version_to_int(max_version)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700493
494 # Check if current bazel version can be detected properly.
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700495 if not curr_version_int:
496 print('WARNING: current bazel installation is not a release version.')
497 print('Make sure you are running at least bazel %s' % min_version)
498 return curr_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700499
Michael Cased94271a2017-08-22 17:26:52 -0700500 print('You have bazel %s installed.' % curr_version)
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700501
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700502 if curr_version_int < min_version_int:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700503 print('Please upgrade your bazel installation to version %s or higher to '
504 'build TensorFlow!' % min_version)
Jason Zamanb41761c2018-10-14 11:28:53 +0800505 sys.exit(1)
TensorFlower Gardener78c246b2018-12-13 12:37:42 -0800506 if (curr_version_int > max_version_int and
507 'TF_IGNORE_MAX_BAZEL_VERSION' not in os.environ):
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800508 print('Please downgrade your bazel installation to version %s or lower to '
Mihai Maruseace0963c42018-12-20 14:27:40 -0800509 'build TensorFlow! To downgrade: download the installer for the old '
510 'version (from https://github.com/bazelbuild/bazel/releases) then '
511 'run the installer.' % max_version)
Jason Zamanb41761c2018-10-14 11:28:53 +0800512 sys.exit(1)
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700513 return curr_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700514
515
516def set_cc_opt_flags(environ_cp):
517 """Set up architecture-dependent optimization flags.
518
519 Also append CC optimization flags to bazel.rc..
520
521 Args:
522 environ_cp: copy of the os.environ.
523 """
524 if is_ppc64le():
525 # gcc on ppc64le does not support -march, use mcpu instead
526 default_cc_opt_flags = '-mcpu=native'
A. Unique TensorFlower1bba94a2018-04-04 15:45:20 -0700527 elif is_windows():
528 default_cc_opt_flags = '/arch:AVX'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700529 else:
Justin Lebar9ef04f52018-10-10 18:52:45 -0700530 default_cc_opt_flags = '-march=native -Wno-sign-compare'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700531 question = ('Please specify optimization flags to use during compilation when'
532 ' bazel option "--config=opt" is specified [Default is %s]: '
533 ) % default_cc_opt_flags
534 cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS',
535 question, default_cc_opt_flags)
536 for opt in cc_opt_flags.split():
Michael Case00177422017-11-10 13:14:03 -0800537 write_to_bazelrc('build:opt --copt=%s' % opt)
538 # It should be safe on the same build host.
A. Unique TensorFlower1bba94a2018-04-04 15:45:20 -0700539 if not is_ppc64le() and not is_windows():
Shanqing Cai71445712018-03-12 19:33:52 -0700540 write_to_bazelrc('build:opt --host_copt=-march=native')
Michael Casebb3355d2017-11-09 08:46:31 -0800541 write_to_bazelrc('build:opt --define with_default_optimizations=true')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700542
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700543
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700544def set_tf_cuda_clang(environ_cp):
545 """set TF_CUDA_CLANG action_env.
546
547 Args:
548 environ_cp: copy of the os.environ.
549 """
550 question = 'Do you want to use clang as CUDA compiler?'
551 yes_reply = 'Clang will be used as CUDA compiler.'
552 no_reply = 'nvcc will be used as CUDA compiler.'
553 set_action_env_var(
554 environ_cp,
555 'TF_CUDA_CLANG',
556 None,
557 False,
558 question=question,
559 yes_reply=yes_reply,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700560 no_reply=no_reply,
561 bazel_config_name='cuda_clang')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700562
563
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800564def set_tf_download_clang(environ_cp):
565 """Set TF_DOWNLOAD_CLANG action_env."""
Ilya Biryukov9e651e42018-03-22 05:33:42 -0700566 question = 'Do you wish to download a fresh release of clang? (Experimental)'
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800567 yes_reply = 'Clang will be downloaded and used to compile tensorflow.'
568 no_reply = 'Clang will not be downloaded.'
569 set_action_env_var(
570 environ_cp,
571 'TF_DOWNLOAD_CLANG',
572 None,
573 False,
574 question=question,
575 yes_reply=yes_reply,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700576 no_reply=no_reply,
577 bazel_config_name='download_clang')
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800578
579
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700580def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var,
581 var_default):
582 """Get var_name either from env, or user or default.
583
584 If var_name has been set as environment variable, use the preset value, else
585 ask for user input. If no input is provided, the default is used.
586
587 Args:
588 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530589 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700590 ask_for_var: string for how to ask for user input.
591 var_default: default value string.
592
593 Returns:
594 string value for var_name
595 """
596 var = environ_cp.get(var_name)
597 if not var:
598 var = get_input(ask_for_var)
Michael Cased94271a2017-08-22 17:26:52 -0700599 print('\n')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700600 if not var:
601 var = var_default
602 return var
603
604
605def set_clang_cuda_compiler_path(environ_cp):
606 """Set CLANG_CUDA_COMPILER_PATH."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700607 default_clang_path = which('clang') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700608 ask_clang_path = ('Please specify which clang should be used as device and '
609 'host compiler. [Default is %s]: ') % default_clang_path
610
611 while True:
612 clang_cuda_compiler_path = get_from_env_or_user_or_default(
613 environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path,
614 default_clang_path)
615 if os.path.exists(clang_cuda_compiler_path):
616 break
617
618 # Reset and retry
619 print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path)
620 environ_cp['CLANG_CUDA_COMPILER_PATH'] = ''
621
622 # Set CLANG_CUDA_COMPILER_PATH
623 environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path
624 write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH',
625 clang_cuda_compiler_path)
626
627
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700628def prompt_loop_or_load_from_env(environ_cp,
629 var_name,
630 var_default,
631 ask_for_var,
632 check_success,
633 error_msg,
634 suppress_default_error=False,
Austin Anderson58b236b2019-10-22 10:36:10 -0700635 resolve_symlinks=False,
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700636 n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS):
Austin Anderson6afface2017-12-05 11:59:17 -0800637 """Loop over user prompts for an ENV param until receiving a valid response.
638
639 For the env param var_name, read from the environment or verify user input
640 until receiving valid input. When done, set var_name in the environ_cp to its
641 new value.
642
643 Args:
644 environ_cp: (Dict) copy of the os.environ.
645 var_name: (String) string for name of environment variable, e.g. "TF_MYVAR".
646 var_default: (String) default value string.
647 ask_for_var: (String) string for how to ask for user input.
648 check_success: (Function) function that takes one argument and returns a
649 boolean. Should return True if the value provided is considered valid. May
650 contain a complex error message if error_msg does not provide enough
651 information. In that case, set suppress_default_error to True.
652 error_msg: (String) String with one and only one '%s'. Formatted with each
653 invalid response upon check_success(input) failure.
654 suppress_default_error: (Bool) Suppress the above error message in favor of
655 one from the check_success function.
Austin Anderson58b236b2019-10-22 10:36:10 -0700656 resolve_symlinks: (Bool) Translate symbolic links into the real filepath.
Austin Anderson6afface2017-12-05 11:59:17 -0800657 n_ask_attempts: (Integer) Number of times to query for valid input before
658 raising an error and quitting.
659
660 Returns:
661 [String] The value of var_name after querying for input.
662
663 Raises:
664 UserInputError: if a query has been attempted n_ask_attempts times without
Frank Chenc4ef9272018-01-10 11:36:52 -0800665 success, assume that the user has made a scripting error, and will
666 continue to provide invalid input. Raise the error to avoid infinitely
667 looping.
Austin Anderson6afface2017-12-05 11:59:17 -0800668 """
669 default = environ_cp.get(var_name) or var_default
670 full_query = '%s [Default is %s]: ' % (
671 ask_for_var,
672 default,
673 )
674
675 for _ in range(n_ask_attempts):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700676 val = get_from_env_or_user_or_default(environ_cp, var_name, full_query,
Austin Anderson6afface2017-12-05 11:59:17 -0800677 default)
678 if check_success(val):
679 break
680 if not suppress_default_error:
681 print(error_msg % val)
682 environ_cp[var_name] = ''
683 else:
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700684 raise UserInputError('Invalid %s setting was provided %d times in a row. '
685 'Assuming to be a scripting mistake.' %
686 (var_name, n_ask_attempts))
Austin Anderson6afface2017-12-05 11:59:17 -0800687
Austin Anderson58b236b2019-10-22 10:36:10 -0700688 if resolve_symlinks and os.path.islink(val):
689 val = os.path.realpath(val)
Austin Anderson6afface2017-12-05 11:59:17 -0800690 environ_cp[var_name] = val
691 return val
692
693
694def create_android_ndk_rule(environ_cp):
695 """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule."""
696 if is_windows() or is_cygwin():
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700697 default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' %
698 environ_cp['APPDATA'])
Austin Anderson6afface2017-12-05 11:59:17 -0800699 elif is_macos():
700 default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME']
701 else:
702 default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME']
703
704 def valid_ndk_path(path):
705 return (os.path.exists(path) and
706 os.path.exists(os.path.join(path, 'source.properties')))
707
708 android_ndk_home_path = prompt_loop_or_load_from_env(
709 environ_cp,
710 var_name='ANDROID_NDK_HOME',
711 var_default=default_ndk_path,
712 ask_for_var='Please specify the home path of the Android NDK to use.',
713 check_success=valid_ndk_path,
714 error_msg=('The path %s or its child file "source.properties" '
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700715 'does not exist.'))
Michael Case51053502018-06-05 17:47:19 -0700716 write_action_env_to_bazelrc('ANDROID_NDK_HOME', android_ndk_home_path)
Jared Dukea0104b72019-04-04 12:23:58 -0700717 write_action_env_to_bazelrc(
718 'ANDROID_NDK_API_LEVEL',
719 get_ndk_api_level(environ_cp, android_ndk_home_path))
Austin Anderson6afface2017-12-05 11:59:17 -0800720
721
722def create_android_sdk_rule(environ_cp):
723 """Set Android variables and write Android SDK WORKSPACE rule."""
724 if is_windows() or is_cygwin():
725 default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA'])
726 elif is_macos():
Shashi Shekharc0ff0cc2018-07-17 09:00:24 -0700727 default_sdk_path = '%s/library/Android/Sdk' % environ_cp['HOME']
Austin Anderson6afface2017-12-05 11:59:17 -0800728 else:
729 default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME']
730
731 def valid_sdk_path(path):
732 return (os.path.exists(path) and
733 os.path.exists(os.path.join(path, 'platforms')) and
734 os.path.exists(os.path.join(path, 'build-tools')))
735
736 android_sdk_home_path = prompt_loop_or_load_from_env(
737 environ_cp,
738 var_name='ANDROID_SDK_HOME',
739 var_default=default_sdk_path,
740 ask_for_var='Please specify the home path of the Android SDK to use.',
741 check_success=valid_sdk_path,
742 error_msg=('Either %s does not exist, or it does not contain the '
743 'subdirectories "platforms" and "build-tools".'))
744
745 platforms = os.path.join(android_sdk_home_path, 'platforms')
746 api_levels = sorted(os.listdir(platforms))
747 api_levels = [x.replace('android-', '') for x in api_levels]
748
749 def valid_api_level(api_level):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700750 return os.path.exists(
751 os.path.join(android_sdk_home_path, 'platforms',
752 'android-' + api_level))
Austin Anderson6afface2017-12-05 11:59:17 -0800753
754 android_api_level = prompt_loop_or_load_from_env(
755 environ_cp,
756 var_name='ANDROID_API_LEVEL',
757 var_default=api_levels[-1],
758 ask_for_var=('Please specify the Android SDK API level to use. '
759 '[Available levels: %s]') % api_levels,
760 check_success=valid_api_level,
761 error_msg='Android-%s is not present in the SDK path.')
762
763 build_tools = os.path.join(android_sdk_home_path, 'build-tools')
764 versions = sorted(os.listdir(build_tools))
765
766 def valid_build_tools(version):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700767 return os.path.exists(
768 os.path.join(android_sdk_home_path, 'build-tools', version))
Austin Anderson6afface2017-12-05 11:59:17 -0800769
770 android_build_tools_version = prompt_loop_or_load_from_env(
771 environ_cp,
772 var_name='ANDROID_BUILD_TOOLS_VERSION',
773 var_default=versions[-1],
774 ask_for_var=('Please specify an Android build tools version to use. '
775 '[Available versions: %s]') % versions,
776 check_success=valid_build_tools,
777 error_msg=('The selected SDK does not have build-tools version %s '
778 'available.'))
779
Michael Case51053502018-06-05 17:47:19 -0700780 write_action_env_to_bazelrc('ANDROID_BUILD_TOOLS_VERSION',
781 android_build_tools_version)
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700782 write_action_env_to_bazelrc('ANDROID_SDK_API_LEVEL', android_api_level)
783 write_action_env_to_bazelrc('ANDROID_SDK_HOME', android_sdk_home_path)
Austin Anderson6afface2017-12-05 11:59:17 -0800784
785
Jared Dukea0104b72019-04-04 12:23:58 -0700786def get_ndk_api_level(environ_cp, android_ndk_home_path):
787 """Gets the appropriate NDK API level to use for the provided Android NDK path."""
788
789 # First check to see if we're using a blessed version of the NDK.
Austin Anderson6afface2017-12-05 11:59:17 -0800790 properties_path = '%s/source.properties' % android_ndk_home_path
791 if is_windows() or is_cygwin():
792 properties_path = cygpath(properties_path)
793 with open(properties_path, 'r') as f:
794 filedata = f.read()
795
796 revision = re.search(r'Pkg.Revision = (\d+)', filedata)
797 if revision:
Jared Dukea0104b72019-04-04 12:23:58 -0700798 ndk_version = revision.group(1)
Michael Case51053502018-06-05 17:47:19 -0700799 else:
800 raise Exception('Unable to parse NDK revision.')
Jared Dukea0104b72019-04-04 12:23:58 -0700801 if int(ndk_version) not in _SUPPORTED_ANDROID_NDK_VERSIONS:
802 print('WARNING: The NDK version in %s is %s, which is not '
803 'supported by Bazel (officially supported versions: %s). Please use '
804 'another version. Compiling Android targets may result in confusing '
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700805 'errors.\n' %
806 (android_ndk_home_path, ndk_version, _SUPPORTED_ANDROID_NDK_VERSIONS))
Jared Dukea0104b72019-04-04 12:23:58 -0700807
808 # Now grab the NDK API level to use. Note that this is different from the
809 # SDK API level, as the NDK API level is effectively the *min* target SDK
810 # version.
811 platforms = os.path.join(android_ndk_home_path, 'platforms')
812 api_levels = sorted(os.listdir(platforms))
813 api_levels = [
814 x.replace('android-', '') for x in api_levels if 'android-' in x
815 ]
816
817 def valid_api_level(api_level):
818 return os.path.exists(
819 os.path.join(android_ndk_home_path, 'platforms',
820 'android-' + api_level))
821
822 android_ndk_api_level = prompt_loop_or_load_from_env(
823 environ_cp,
824 var_name='ANDROID_NDK_API_LEVEL',
Jared Dukea1309f42019-11-15 17:09:35 -0800825 var_default='21', # 21 is required for ARM64 support.
Jared Dukea0104b72019-04-04 12:23:58 -0700826 ask_for_var=('Please specify the (min) Android NDK API level to use. '
827 '[Available levels: %s]') % api_levels,
828 check_success=valid_api_level,
829 error_msg='Android-%s is not present in the NDK path.')
830
831 return android_ndk_api_level
Austin Anderson6afface2017-12-05 11:59:17 -0800832
833
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700834def set_gcc_host_compiler_path(environ_cp):
835 """Set GCC_HOST_COMPILER_PATH."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700836 default_gcc_host_compiler_path = which('gcc') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700837 cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH')
838
839 if os.path.islink(cuda_bin_symlink):
840 # os.readlink is only available in linux
Jonathan Hseu008910f2017-08-25 14:01:05 -0700841 default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700842
Austin Anderson6afface2017-12-05 11:59:17 -0800843 gcc_host_compiler_path = prompt_loop_or_load_from_env(
844 environ_cp,
845 var_name='GCC_HOST_COMPILER_PATH',
846 var_default=default_gcc_host_compiler_path,
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800847 ask_for_var='Please specify which gcc should be used by nvcc as the host compiler.',
Austin Anderson6afface2017-12-05 11:59:17 -0800848 check_success=os.path.exists,
Austin Anderson58b236b2019-10-22 10:36:10 -0700849 resolve_symlinks=True,
Austin Anderson6afface2017-12-05 11:59:17 -0800850 error_msg='Invalid gcc path. %s cannot be found.',
851 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700852
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700853 write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path)
854
855
Ankur Taly0e6f39d2018-02-16 18:22:55 -0800856def reformat_version_sequence(version_str, sequence_count):
857 """Reformat the version string to have the given number of sequences.
858
859 For example:
860 Given (7, 2) -> 7.0
861 (7.0.1, 2) -> 7.0
862 (5, 1) -> 5
863 (5.0.3.2, 1) -> 5
864
865 Args:
866 version_str: String, the version string.
867 sequence_count: int, an integer.
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700868
Ankur Taly0e6f39d2018-02-16 18:22:55 -0800869 Returns:
870 string, reformatted version string.
871 """
872 v = version_str.split('.')
873 if len(v) < sequence_count:
874 v = v + (['0'] * (sequence_count - len(v)))
875
876 return '.'.join(v[:sequence_count])
877
878
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700879def set_tf_cuda_paths(environ_cp):
880 """Set TF_CUDA_PATHS."""
881 ask_cuda_paths = (
882 'Please specify the comma-separated list of base paths to look for CUDA '
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700883 'libraries and headers. [Leave empty to use the default]: ')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700884 tf_cuda_paths = get_from_env_or_user_or_default(environ_cp, 'TF_CUDA_PATHS',
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700885 ask_cuda_paths, '')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700886 if tf_cuda_paths:
887 environ_cp['TF_CUDA_PATHS'] = tf_cuda_paths
888
889
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700890def set_tf_cuda_version(environ_cp):
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700891 """Set TF_CUDA_VERSION."""
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700892 ask_cuda_version = (
893 'Please specify the CUDA SDK version you want to use. '
894 '[Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700895 tf_cuda_version = get_from_env_or_user_or_default(environ_cp,
896 'TF_CUDA_VERSION',
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700897 ask_cuda_version,
898 _DEFAULT_CUDA_VERSION)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700899 environ_cp['TF_CUDA_VERSION'] = tf_cuda_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700900
901
Yifei Fengb1d8c592017-11-22 13:42:21 -0800902def set_tf_cudnn_version(environ_cp):
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700903 """Set TF_CUDNN_VERSION."""
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700904 ask_cudnn_version = (
905 'Please specify the cuDNN version you want to use. '
906 '[Leave empty to default to cuDNN %s]: ') % _DEFAULT_CUDNN_VERSION
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700907 tf_cudnn_version = get_from_env_or_user_or_default(environ_cp,
908 'TF_CUDNN_VERSION',
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700909 ask_cudnn_version,
910 _DEFAULT_CUDNN_VERSION)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700911 environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700912
913
Mingxing Tan1e7b0e42018-06-28 19:13:20 -0700914def is_cuda_compatible(lib, cuda_ver, cudnn_ver):
915 """Check compatibility between given library and cudnn/cudart libraries."""
916 ldd_bin = which('ldd') or '/usr/bin/ldd'
917 ldd_out = run_shell([ldd_bin, lib], True)
918 ldd_out = ldd_out.split(os.linesep)
919 cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$')
920 cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$')
921 cudnn = None
922 cudart = None
923 cudnn_ok = True # assume no cudnn dependency by default
924 cuda_ok = True # assume no cuda dependency by default
925 for line in ldd_out:
926 if 'libcudnn.so' in line:
927 cudnn = cudnn_pattern.search(line)
928 cudnn_ok = False
929 elif 'libcudart.so' in line:
930 cudart = cuda_pattern.search(line)
931 cuda_ok = False
932 if cudnn and len(cudnn.group(1)):
933 cudnn = convert_version_to_int(cudnn.group(1))
934 if cudart and len(cudart.group(1)):
935 cudart = convert_version_to_int(cudart.group(1))
936 if cudnn is not None:
937 cudnn_ok = (cudnn == cudnn_ver)
938 if cudart is not None:
939 cuda_ok = (cudart == cuda_ver)
940 return cudnn_ok and cuda_ok
941
942
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700943def set_tf_tensorrt_version(environ_cp):
944 """Set TF_TENSORRT_VERSION."""
Guangda Lai76f69382018-01-25 23:59:19 -0800945 if not is_linux():
946 raise ValueError('Currently TensorRT is only supported on Linux platform.')
947
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700948 if not int(environ_cp.get('TF_NEED_TENSORRT', False)):
Guangda Lai76f69382018-01-25 23:59:19 -0800949 return
950
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700951 ask_tensorrt_version = (
952 'Please specify the TensorRT version you want to use. '
Guangda Laifaa93ac2019-10-23 15:07:12 -0700953 '[Leave empty to default to TensorRT %s]: ') % _DEFAULT_TENSORRT_VERSION
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700954 tf_tensorrt_version = get_from_env_or_user_or_default(
955 environ_cp, 'TF_TENSORRT_VERSION', ask_tensorrt_version,
956 _DEFAULT_TENSORRT_VERSION)
Guangda Lai76f69382018-01-25 23:59:19 -0800957 environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version
Guangda Lai76f69382018-01-25 23:59:19 -0800958
959
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700960def set_tf_nccl_version(environ_cp):
961 """Set TF_NCCL_VERSION."""
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700962 if not is_linux():
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700963 raise ValueError('Currently NCCL is only supported on Linux platform.')
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700964
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700965 if 'TF_NCCL_VERSION' in environ_cp:
966 return
967
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700968 ask_nccl_version = (
A. Unique TensorFlower53faa312018-10-05 08:46:54 -0700969 'Please specify the locally installed NCCL version you want to use. '
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700970 '[Leave empty to use http://github.com/nvidia/nccl]: ')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700971 tf_nccl_version = get_from_env_or_user_or_default(environ_cp,
972 'TF_NCCL_VERSION',
973 ask_nccl_version, '')
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700974 environ_cp['TF_NCCL_VERSION'] = tf_nccl_version
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800975
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700976
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700977def get_native_cuda_compute_capabilities(environ_cp):
978 """Get native cuda compute capabilities.
979
980 Args:
981 environ_cp: copy of the os.environ.
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700982
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700983 Returns:
984 string of native cuda compute capabilities, separated by comma.
985 """
986 device_query_bin = os.path.join(
987 environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery')
Jonathan Hseu008910f2017-08-25 14:01:05 -0700988 if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK):
989 try:
990 output = run_shell(device_query_bin).split('\n')
991 pattern = re.compile('[0-9]*\\.[0-9]*')
992 output = [pattern.search(x) for x in output if 'Capability' in x]
993 output = ','.join(x.group() for x in output if x is not None)
994 except subprocess.CalledProcessError:
995 output = ''
996 else:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700997 output = ''
998 return output
999
1000
1001def set_tf_cuda_compute_capabilities(environ_cp):
1002 """Set TF_CUDA_COMPUTE_CAPABILITIES."""
1003 while True:
1004 native_cuda_compute_capabilities = get_native_cuda_compute_capabilities(
1005 environ_cp)
1006 if not native_cuda_compute_capabilities:
1007 default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES
1008 else:
1009 default_cuda_compute_capabilities = native_cuda_compute_capabilities
1010
1011 ask_cuda_compute_capabilities = (
1012 'Please specify a list of comma-separated '
P Sudeepam52093562019-02-17 17:34:01 +05301013 'CUDA compute capabilities you want to '
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001014 'build with.\nYou can find the compute '
1015 'capability of your device at: '
1016 'https://developer.nvidia.com/cuda-gpus.\nPlease'
1017 ' note that each additional compute '
1018 'capability significantly increases your '
P Sudeepam52093562019-02-17 17:34:01 +05301019 'build time and binary size, and that '
1020 'TensorFlow only supports compute '
P Sudeepam765ceda2019-02-17 17:39:08 +05301021 'capabilities >= 3.5 [Default is: %s]: ' %
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001022 default_cuda_compute_capabilities)
1023 tf_cuda_compute_capabilities = get_from_env_or_user_or_default(
1024 environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES',
1025 ask_cuda_compute_capabilities, default_cuda_compute_capabilities)
1026 # Check whether all capabilities from the input is valid
1027 all_valid = True
Maciejd0f5bc12018-04-30 22:30:58 -05001028 # Remove all whitespace characters before splitting the string
Michael Case51053502018-06-05 17:47:19 -07001029 # that users may insert by accident, as this will result in error
Maciejd0f5bc12018-04-30 22:30:58 -05001030 tf_cuda_compute_capabilities = ''.join(tf_cuda_compute_capabilities.split())
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001031 for compute_capability in tf_cuda_compute_capabilities.split(','):
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001032 m = re.match('[0-9]+.[0-9]+', compute_capability)
1033 if not m:
Austin Anderson32202dc2019-02-19 10:46:27 -08001034 print('Invalid compute capability: %s' % compute_capability)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001035 all_valid = False
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001036 else:
P Sudeepam52093562019-02-17 17:34:01 +05301037 ver = float(m.group(0))
A. Unique TensorFlower8dc2d0e2019-03-12 01:41:05 -07001038 if ver < 3.0:
1039 print('ERROR: TensorFlow only supports CUDA compute capabilities 3.0 '
Austin Anderson32202dc2019-02-19 10:46:27 -08001040 'and higher. Please re-specify the list of compute '
1041 'capabilities excluding version %s.' % ver)
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001042 all_valid = False
A. Unique TensorFlower8dc2d0e2019-03-12 01:41:05 -07001043 if ver < 3.5:
1044 print('WARNING: XLA does not support CUDA compute capabilities '
1045 'lower than 3.5. Disable XLA when running on older GPUs.')
P Sudeepam765ceda2019-02-17 17:39:08 +05301046
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001047 if all_valid:
1048 break
1049
1050 # Reset and Retry
1051 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = ''
1052
1053 # Set TF_CUDA_COMPUTE_CAPABILITIES
1054 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities
1055 write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES',
1056 tf_cuda_compute_capabilities)
1057
1058
1059def set_other_cuda_vars(environ_cp):
1060 """Set other CUDA related variables."""
A. Unique TensorFlowerab391982018-07-11 04:52:49 -07001061 # If CUDA is enabled, always use GPU during build and test.
1062 if environ_cp.get('TF_CUDA_CLANG') == '1':
1063 write_to_bazelrc('build --config=cuda_clang')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001064 else:
A. Unique TensorFlowerab391982018-07-11 04:52:49 -07001065 write_to_bazelrc('build --config=cuda')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001066
1067
1068def set_host_cxx_compiler(environ_cp):
1069 """Set HOST_CXX_COMPILER."""
Jonathan Hseu008910f2017-08-25 14:01:05 -07001070 default_cxx_host_compiler = which('g++') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001071
Austin Anderson6afface2017-12-05 11:59:17 -08001072 host_cxx_compiler = prompt_loop_or_load_from_env(
1073 environ_cp,
1074 var_name='HOST_CXX_COMPILER',
1075 var_default=default_cxx_host_compiler,
1076 ask_for_var=('Please specify which C++ compiler should be used as the '
1077 'host C++ compiler.'),
1078 check_success=os.path.exists,
1079 error_msg='Invalid C++ compiler path. %s cannot be found.',
1080 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001081
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001082 write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler)
1083
1084
1085def set_host_c_compiler(environ_cp):
1086 """Set HOST_C_COMPILER."""
Jonathan Hseu008910f2017-08-25 14:01:05 -07001087 default_c_host_compiler = which('gcc') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001088
Austin Anderson6afface2017-12-05 11:59:17 -08001089 host_c_compiler = prompt_loop_or_load_from_env(
1090 environ_cp,
1091 var_name='HOST_C_COMPILER',
1092 var_default=default_c_host_compiler,
Shanqing Cai71445712018-03-12 19:33:52 -07001093 ask_for_var=('Please specify which C compiler should be used as the host '
Austin Anderson6afface2017-12-05 11:59:17 -08001094 'C compiler.'),
1095 check_success=os.path.exists,
1096 error_msg='Invalid C compiler path. %s cannot be found.',
1097 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001098
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001099 write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler)
1100
1101
1102def set_computecpp_toolkit_path(environ_cp):
1103 """Set COMPUTECPP_TOOLKIT_PATH."""
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001104
Austin Anderson6afface2017-12-05 11:59:17 -08001105 def toolkit_exists(toolkit_path):
1106 """Check if a computecpp toolkit path is valid."""
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001107 if is_linux():
1108 sycl_rt_lib_path = 'lib/libComputeCpp.so'
1109 else:
1110 sycl_rt_lib_path = ''
1111
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001112 sycl_rt_lib_path_full = os.path.join(toolkit_path, sycl_rt_lib_path)
Austin Anderson6afface2017-12-05 11:59:17 -08001113 exists = os.path.exists(sycl_rt_lib_path_full)
1114 if not exists:
1115 print('Invalid SYCL %s library path. %s cannot be found' %
1116 (_TF_OPENCL_VERSION, sycl_rt_lib_path_full))
1117 return exists
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001118
Austin Anderson6afface2017-12-05 11:59:17 -08001119 computecpp_toolkit_path = prompt_loop_or_load_from_env(
1120 environ_cp,
1121 var_name='COMPUTECPP_TOOLKIT_PATH',
1122 var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH,
1123 ask_for_var=(
1124 'Please specify the location where ComputeCpp for SYCL %s is '
1125 'installed.' % _TF_OPENCL_VERSION),
1126 check_success=toolkit_exists,
1127 error_msg='Invalid SYCL compiler path. %s cannot be found.',
1128 suppress_default_error=True)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001129
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001130 write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH',
1131 computecpp_toolkit_path)
1132
Michael Cased31531a2018-01-05 14:09:41 -08001133
Dandelion Man?90e42f32017-12-15 18:15:07 -08001134def set_trisycl_include_dir(environ_cp):
Michael Cased31531a2018-01-05 14:09:41 -08001135 """Set TRISYCL_INCLUDE_DIR."""
Frank Chenc4ef9272018-01-10 11:36:52 -08001136
Dandelion Man?90e42f32017-12-15 18:15:07 -08001137 ask_trisycl_include_dir = ('Please specify the location of the triSYCL '
1138 'include directory. (Use --config=sycl_trisycl '
1139 'when building with Bazel) '
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001140 '[Default is %s]: ') % (
1141 _DEFAULT_TRISYCL_INCLUDE_DIR)
Frank Chenc4ef9272018-01-10 11:36:52 -08001142
Dandelion Man?90e42f32017-12-15 18:15:07 -08001143 while True:
1144 trisycl_include_dir = get_from_env_or_user_or_default(
Michael Cased31531a2018-01-05 14:09:41 -08001145 environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir,
1146 _DEFAULT_TRISYCL_INCLUDE_DIR)
Dandelion Man?90e42f32017-12-15 18:15:07 -08001147 if os.path.exists(trisycl_include_dir):
1148 break
1149
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001150 print('Invalid triSYCL include directory, %s cannot be found' %
1151 (trisycl_include_dir))
Dandelion Man?90e42f32017-12-15 18:15:07 -08001152
1153 # Set TRISYCL_INCLUDE_DIR
1154 environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001155 write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', trisycl_include_dir)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001156
Yifei Fengb1d8c592017-11-22 13:42:21 -08001157
A. Unique TensorFlowere9797fd2019-03-22 11:02:40 -07001158def system_specific_test_config(env):
A. Unique TensorFlower7bd86372019-03-21 15:19:30 -07001159 """Add default build and test flags required for TF tests to bazelrc."""
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001160 write_to_bazelrc('test --flaky_test_attempts=3')
1161 write_to_bazelrc('test --test_size_filters=small,medium')
frreiss8b013a72019-11-20 11:36:27 -08001162
1163 # Each instance of --test_tag_filters or --build_tag_filters overrides all
1164 # previous instances, so we need to build up a complete list and write a
1165 # single list of filters for the .bazelrc file.
1166
1167 # Filters to use with both --test_tag_filters and --build_tag_filters
frreissbab7d7d2019-11-21 10:52:55 -08001168 test_and_build_filters = ['-benchmark-test', '-no_oss']
1169 # Additional filters for --test_tag_filters beyond those in
1170 # test_and_build_filters
1171 test_only_filters = ['-oss_serial']
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001172 if is_windows():
frreissbab7d7d2019-11-21 10:52:55 -08001173 test_and_build_filters.append('-no_windows')
Guangda Laibcd701a2019-03-12 21:04:51 -07001174 if env.get('TF_NEED_CUDA', None) == '1':
frreissbab7d7d2019-11-21 10:52:55 -08001175 test_and_build_filters += ['-no_windows_gpu', '-no_gpu']
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001176 else:
frreissbab7d7d2019-11-21 10:52:55 -08001177 test_and_build_filters.append('-gpu')
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001178 elif is_macos():
frreissbab7d7d2019-11-21 10:52:55 -08001179 test_and_build_filters += ['-gpu', '-nomac', '-no_mac']
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001180 elif is_linux():
Guangda Laibcd701a2019-03-12 21:04:51 -07001181 if env.get('TF_NEED_CUDA', None) == '1':
frreissbab7d7d2019-11-21 10:52:55 -08001182 test_and_build_filters.append('-no_gpu')
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001183 write_to_bazelrc('test --test_env=LD_LIBRARY_PATH')
1184 else:
frreissbab7d7d2019-11-21 10:52:55 -08001185 test_and_build_filters.append('-gpu')
frreiss3adf4172019-12-06 14:16:49 -08001186
1187 # Disable tests with "v1only" tag in "v2" Bazel config, but not in "v1" config
1188 write_to_bazelrc('test:v1 --test_tag_filters=%s' %
TensorFlower Gardenere5159082019-11-22 11:47:38 -08001189 ','.join(test_and_build_filters + test_only_filters))
frreiss3adf4172019-12-06 14:16:49 -08001190 write_to_bazelrc('test:v1 --build_tag_filters=%s' %
TensorFlower Gardenere5159082019-11-22 11:47:38 -08001191 ','.join(test_and_build_filters))
TensorFlower Gardener26a1de32019-12-17 07:11:48 -08001192 write_to_bazelrc(
1193 'test:v2 --test_tag_filters=%s' %
1194 ','.join(test_and_build_filters + test_only_filters + ['-v1only']))
frreiss3adf4172019-12-06 14:16:49 -08001195 write_to_bazelrc('test:v2 --build_tag_filters=%s' %
TensorFlower Gardener26a1de32019-12-17 07:11:48 -08001196 ','.join(test_and_build_filters + ['-v1only']))
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001197
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001198
Yifei Feng5198cb82018-08-17 13:53:06 -07001199def set_system_libs_flag(environ_cp):
1200 syslibs = environ_cp.get('TF_SYSTEM_LIBS', '')
TensorFlower Gardener61a87202018-10-01 12:25:39 -07001201 if syslibs:
Jason Zaman5fc39bd2018-09-16 01:38:55 +08001202 if ',' in syslibs:
1203 syslibs = ','.join(sorted(syslibs.split(',')))
1204 else:
1205 syslibs = ','.join(sorted(syslibs.split()))
Yifei Feng5198cb82018-08-17 13:53:06 -07001206 write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs)
1207
Jason Zaman5fc39bd2018-09-16 01:38:55 +08001208 if 'PREFIX' in environ_cp:
1209 write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX'])
1210 if 'LIBDIR' in environ_cp:
1211 write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR'])
1212 if 'INCLUDEDIR' in environ_cp:
1213 write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR'])
1214
Yifei Feng5198cb82018-08-17 13:53:06 -07001215
Austin Anderson6d7c3762019-11-19 17:04:19 -08001216def is_reduced_optimize_huge_functions_available(environ_cp):
1217 """Check to see if the system supports /d2ReducedOptimizeHugeFunctions.
1218
1219 The above compiler flag is a new compiler flag introduced to the Visual Studio
1220 compiler in version 16.4 (available in Visual Studio 2019, Preview edition
1221 only, as of 2019-11-19). TensorFlow needs this flag to massively reduce
1222 compile times, but until 16.4 is officially released, we can't depend on it.
1223
Ahti Kitsik1dc9a3c2020-01-22 11:32:48 +01001224 See also https://groups.google.com/a/tensorflow.org/d/topic/build/SsW98Eo7l3o/discussion
Austin Anderson6d7c3762019-11-19 17:04:19 -08001225
1226 Because it's very annoying to check this manually (to check the MSVC installed
1227 versions, you need to use the registry, and it's not clear if Bazel will be
1228 using that install version anyway), we expect enviroments who know they may
1229 use this flag to export TF_VC_VERSION=16.4
1230
1231 TODO(angerson, gunan): Remove this function when TensorFlow's minimum VS
1232 version is upgraded to 16.4.
1233
1234 Arguments:
1235 environ_cp: Environment of the current execution
1236
1237 Returns:
1238 boolean, whether or not /d2ReducedOptimizeHugeFunctions is available on this
1239 machine.
1240 """
1241 return float(environ_cp.get('TF_VC_VERSION', '0')) >= 16.4
1242
1243
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001244def set_windows_build_flags(environ_cp):
1245 """Set Windows specific build options."""
Austin Anderson6d7c3762019-11-19 17:04:19 -08001246 if is_reduced_optimize_huge_functions_available(environ_cp):
1247 write_to_bazelrc(
1248 'build --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions'
1249 )
1250
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001251 if get_var(
1252 environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline',
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001253 True, ('Would you like to override eigen strong inline for some C++ '
1254 'compilation to reduce the compilation time?'),
1255 'Eigen strong inline overridden.', 'Not overriding eigen strong inline, '
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001256 'some compilations could take more than 20 mins.'):
1257 # Due to a known MSVC compiler issue
1258 # https://github.com/tensorflow/tensorflow/issues/10521
1259 # Overriding eigen strong inline speeds up the compiling of
1260 # conv_grad_ops_3d.cc and conv_ops_3d.cc by 20 minutes,
1261 # but this also hurts the performance. Let users decide what they want.
1262 write_to_bazelrc('build --define=override_eigen_strong_inline=true')
Dandelion Man?90e42f32017-12-15 18:15:07 -08001263
A. Unique TensorFlower061c3592017-11-13 14:21:04 -08001264
Michael Cased31531a2018-01-05 14:09:41 -08001265def config_info_line(name, help_text):
1266 """Helper function to print formatted help text for Bazel config options."""
1267 print('\t--config=%-12s\t# %s' % (name, help_text))
1268
1269
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001270def configure_ios():
1271 """Configures TensorFlow for iOS builds.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001272
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001273 This function will only be executed if `is_macos()` is true.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001274 """
1275 if not is_macos():
1276 return
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001277 for filepath in APPLE_BAZEL_FILES:
1278 existing_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath + '.apple')
1279 renamed_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath)
1280 symlink_force(existing_filepath, renamed_filepath)
1281 for filepath in IOS_FILES:
1282 filename = os.path.basename(filepath)
1283 new_filepath = os.path.join(_TF_WORKSPACE_ROOT, filename)
1284 symlink_force(filepath, new_filepath)
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001285
1286
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001287def validate_cuda_config(environ_cp):
1288 """Run find_cuda_config.py and return cuda_toolkit_path, or None."""
1289
1290 def maybe_encode_env(env):
1291 """Encodes unicode in env to str on Windows python 2.x."""
1292 if not is_windows() or sys.version_info[0] != 2:
1293 return env
1294 for k, v in env.items():
1295 if isinstance(k, unicode):
1296 k = k.encode('ascii')
1297 if isinstance(v, unicode):
1298 v = v.encode('ascii')
1299 env[k] = v
1300 return env
1301
1302 cuda_libraries = ['cuda', 'cudnn']
1303 if is_linux():
A. Unique TensorFlower28fc9cc2019-05-01 14:17:54 -07001304 if int(environ_cp.get('TF_NEED_TENSORRT', False)):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001305 cuda_libraries.append('tensorrt')
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001306 if environ_cp.get('TF_NCCL_VERSION', None):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001307 cuda_libraries.append('nccl')
1308
1309 proc = subprocess.Popen(
1310 [environ_cp['PYTHON_BIN_PATH'], 'third_party/gpus/find_cuda_config.py'] +
1311 cuda_libraries,
1312 stdout=subprocess.PIPE,
1313 env=maybe_encode_env(environ_cp))
1314
1315 if proc.wait():
1316 # Errors from find_cuda_config.py were sent to stderr.
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -07001317 print('Asking for detailed CUDA configuration...\n')
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001318 return False
1319
1320 config = dict(
1321 tuple(line.decode('ascii').rstrip().split(': ')) for line in proc.stdout)
1322
1323 print('Found CUDA %s in:' % config['cuda_version'])
1324 print(' %s' % config['cuda_library_dir'])
1325 print(' %s' % config['cuda_include_dir'])
1326
1327 print('Found cuDNN %s in:' % config['cudnn_version'])
1328 print(' %s' % config['cudnn_library_dir'])
1329 print(' %s' % config['cudnn_include_dir'])
1330
1331 if 'tensorrt_version' in config:
1332 print('Found TensorRT %s in:' % config['tensorrt_version'])
1333 print(' %s' % config['tensorrt_library_dir'])
1334 print(' %s' % config['tensorrt_include_dir'])
1335
1336 if config.get('nccl_version', None):
1337 print('Found NCCL %s in:' % config['nccl_version'])
1338 print(' %s' % config['nccl_library_dir'])
1339 print(' %s' % config['nccl_include_dir'])
1340
1341 print('\n')
1342
1343 environ_cp['CUDA_TOOLKIT_PATH'] = config['cuda_toolkit_path']
1344 return True
1345
1346
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001347def main():
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001348 global _TF_WORKSPACE_ROOT
1349 global _TF_BAZELRC
A. Unique TensorFlowered297342019-03-15 11:25:28 -07001350 global _TF_CURRENT_BAZEL_VERSION
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001351
Shanqing Cai71445712018-03-12 19:33:52 -07001352 parser = argparse.ArgumentParser()
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001353 parser.add_argument(
1354 '--workspace',
1355 type=str,
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001356 default=os.path.abspath(os.path.dirname(__file__)),
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001357 help='The absolute path to your active Bazel workspace.')
Shanqing Cai71445712018-03-12 19:33:52 -07001358 args = parser.parse_args()
1359
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001360 _TF_WORKSPACE_ROOT = args.workspace
1361 _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME)
1362
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001363 # Make a copy of os.environ to be clear when functions and getting and setting
1364 # environment variables.
1365 environ_cp = dict(os.environ)
1366
Mark Daoust44000ad2019-06-18 09:26:26 -07001367 current_bazel_version = check_bazel_version(_TF_MIN_BAZEL_VERSION,
1368 _TF_MAX_BAZEL_VERSION)
A. Unique TensorFlowered297342019-03-15 11:25:28 -07001369 _TF_CURRENT_BAZEL_VERSION = convert_version_to_int(current_bazel_version)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001370
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001371 reset_tf_configure_bazelrc()
Yun Peng03e63a22018-11-07 11:18:53 +01001372
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001373 cleanup_makefile()
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -07001374 setup_python(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001375
1376 if is_windows():
Yifei Fengb1d8c592017-11-22 13:42:21 -08001377 environ_cp['TF_NEED_OPENCL_SYCL'] = '0'
1378 environ_cp['TF_NEED_COMPUTECPP'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001379 environ_cp['TF_NEED_OPENCL'] = '0'
1380 environ_cp['TF_CUDA_CLANG'] = '0'
Guangda Lai76f69382018-01-25 23:59:19 -08001381 environ_cp['TF_NEED_TENSORRT'] = '0'
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001382 # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on
1383 # Windows.
1384 environ_cp['TF_DOWNLOAD_CLANG'] = '0'
A. Unique TensorFlower6e97fb32018-07-16 14:07:29 -07001385 environ_cp['TF_NEED_MPI'] = '0'
1386 environ_cp['TF_SET_ANDROID_WORKSPACE'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001387
1388 if is_macos():
Guangda Lai76f69382018-01-25 23:59:19 -08001389 environ_cp['TF_NEED_TENSORRT'] = '0'
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001390 else:
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001391 environ_cp['TF_CONFIGURE_IOS'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001392
Sanjoy Dasf3fab7c2019-06-20 11:19:00 -07001393 xla_enabled_by_default = is_linux() or is_macos()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001394 set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
Grzegorz Pawelczakec82efd2018-10-09 15:03:46 +01001395 xla_enabled_by_default, 'xla')
Gunhan Gulsoy3da0dff2018-09-26 11:55:50 -07001396
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001397 set_action_env_var(
1398 environ_cp,
1399 'TF_NEED_OPENCL_SYCL',
1400 'OpenCL SYCL',
1401 False,
1402 bazel_config_name='sycl')
Yifei Fengb1d8c592017-11-22 13:42:21 -08001403 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1':
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001404 set_host_cxx_compiler(environ_cp)
1405 set_host_c_compiler(environ_cp)
Yifei Fengb1d8c592017-11-22 13:42:21 -08001406 set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True)
1407 if environ_cp.get('TF_NEED_COMPUTECPP') == '1':
1408 set_computecpp_toolkit_path(environ_cp)
1409 else:
1410 set_trisycl_include_dir(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001411
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001412 set_action_env_var(
1413 environ_cp, 'TF_NEED_ROCM', 'ROCm', False, bazel_config_name='rocm')
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001414 if (environ_cp.get('TF_NEED_ROCM') == '1' and
TensorFlower Gardener62e60162018-09-27 10:22:55 -07001415 'LD_LIBRARY_PATH' in environ_cp and
1416 environ_cp.get('LD_LIBRARY_PATH') != '1'):
1417 write_action_env_to_bazelrc('LD_LIBRARY_PATH',
1418 environ_cp.get('LD_LIBRARY_PATH'))
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001419
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001420 environ_cp['TF_NEED_CUDA'] = str(
1421 int(get_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False)))
A. Unique TensorFlower24cbb2a2017-09-08 07:45:44 -07001422 if (environ_cp.get('TF_NEED_CUDA') == '1' and
1423 'TF_CUDA_CONFIG_REPO' not in environ_cp):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001424
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001425 set_action_env_var(
1426 environ_cp,
1427 'TF_NEED_TENSORRT',
1428 'TensorRT',
1429 False,
1430 bazel_config_name='tensorrt')
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001431
1432 environ_save = dict(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001433 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001434
1435 if validate_cuda_config(environ_cp):
1436 cuda_env_names = [
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001437 'TF_CUDA_VERSION',
1438 'TF_CUBLAS_VERSION',
1439 'TF_CUDNN_VERSION',
1440 'TF_TENSORRT_VERSION',
1441 'TF_NCCL_VERSION',
1442 'TF_CUDA_PATHS',
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001443 # Items below are for backwards compatibility when not using
1444 # TF_CUDA_PATHS.
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001445 'CUDA_TOOLKIT_PATH',
1446 'CUDNN_INSTALL_PATH',
1447 'NCCL_INSTALL_PATH',
1448 'NCCL_HDR_PATH',
1449 'TENSORRT_INSTALL_PATH'
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001450 ]
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001451 # Note: set_action_env_var above already writes to bazelrc.
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001452 for name in cuda_env_names:
1453 if name in environ_cp:
1454 write_action_env_to_bazelrc(name, environ_cp[name])
1455 break
1456
1457 # Restore settings changed below if CUDA config could not be validated.
1458 environ_cp = dict(environ_save)
1459
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001460 set_tf_cuda_version(environ_cp)
1461 set_tf_cudnn_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001462 if is_linux():
1463 set_tf_tensorrt_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001464 set_tf_nccl_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001465
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001466 set_tf_cuda_paths(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001467
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001468 else:
1469 raise UserInputError(
1470 'Invalid CUDA setting were provided %d '
1471 'times in a row. Assuming to be a scripting mistake.' %
1472 _DEFAULT_PROMPT_ASK_ATTEMPTS)
1473
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001474 set_tf_cuda_compute_capabilities(environ_cp)
Ankur Taly0e6f39d2018-02-16 18:22:55 -08001475 if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get(
1476 'LD_LIBRARY_PATH') != '1':
1477 write_action_env_to_bazelrc('LD_LIBRARY_PATH',
1478 environ_cp.get('LD_LIBRARY_PATH'))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001479
1480 set_tf_cuda_clang(environ_cp)
1481 if environ_cp.get('TF_CUDA_CLANG') == '1':
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001482 # Ask whether we should download the clang toolchain.
1483 set_tf_download_clang(environ_cp)
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -08001484 if environ_cp.get('TF_DOWNLOAD_CLANG') != '1':
1485 # Set up which clang we should use as the cuda / host compiler.
1486 set_clang_cuda_compiler_path(environ_cp)
Ilya Biryukov1c3d02e2018-09-04 03:09:52 -07001487 else:
1488 # Use downloaded LLD for linking.
1489 write_to_bazelrc('build:cuda_clang --config=download_clang_use_lld')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001490 else:
1491 # Set up which gcc nvcc should use as the host compiler
1492 # No need to set this on Windows
1493 if not is_windows():
1494 set_gcc_host_compiler_path(environ_cp)
1495 set_other_cuda_vars(environ_cp)
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001496 else:
1497 # CUDA not required. Ask whether we should download the clang toolchain and
1498 # use it for the CPU build.
1499 set_tf_download_clang(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001500
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001501 # SYCL / ROCm / CUDA are mutually exclusive.
1502 # At most 1 GPU platform can be configured.
1503 gpu_platform_count = 0
1504 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1':
1505 gpu_platform_count += 1
1506 if environ_cp.get('TF_NEED_ROCM') == '1':
1507 gpu_platform_count += 1
1508 if environ_cp.get('TF_NEED_CUDA') == '1':
1509 gpu_platform_count += 1
1510 if gpu_platform_count >= 2:
1511 raise UserInputError('SYCL / CUDA / ROCm are mututally exclusive. '
1512 'At most 1 GPU platform can be configured.')
1513
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001514 set_cc_opt_flags(environ_cp)
Yifei Feng5198cb82018-08-17 13:53:06 -07001515 set_system_libs_flag(environ_cp)
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001516 if is_windows():
1517 set_windows_build_flags(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001518
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001519 if get_var(environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', False,
1520 ('Would you like to interactively configure ./WORKSPACE for '
1521 'Android builds?'), 'Searching for NDK and SDK installations.',
1522 'Not configuring the WORKSPACE for Android builds.'):
Michael Case51053502018-06-05 17:47:19 -07001523 create_android_ndk_rule(environ_cp)
1524 create_android_sdk_rule(environ_cp)
Austin Anderson6afface2017-12-05 11:59:17 -08001525
A. Unique TensorFlowere9797fd2019-03-22 11:02:40 -07001526 system_specific_test_config(os.environ)
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001527
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -07001528 set_action_env_var(environ_cp, 'TF_CONFIGURE_IOS', 'iOS', False)
1529 if environ_cp.get('TF_CONFIGURE_IOS') == '1':
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001530 configure_ios()
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001531
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001532 print('Preconfigured Bazel build configs. You can use any of the below by '
1533 'adding "--config=<>" to your build command. See .bazelrc for more '
1534 'details.')
1535 config_info_line('mkl', 'Build with MKL support.')
1536 config_info_line('monolithic', 'Config for mostly static monolithic build.')
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001537 config_info_line('ngraph', 'Build with Intel nGraph support.')
A. Unique TensorFlowera6bf9c82019-02-26 10:08:35 -08001538 config_info_line('numa', 'Build with NUMA support.')
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -08001539 config_info_line(
1540 'dynamic_kernels',
1541 '(Experimental) Build kernels into separate shared objects.')
Anna Raedf6742019-06-12 11:30:51 -07001542 config_info_line('v2', 'Build TensorFlow 2.x instead of 1.x.')
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001543
1544 print('Preconfigured Bazel build configs to DISABLE default on features:')
1545 config_info_line('noaws', 'Disable AWS S3 filesystem support.')
1546 config_info_line('nogcp', 'Disable GCP support.')
1547 config_info_line('nohdfs', 'Disable HDFS support.')
Gunhan Gulsoyeea81682018-11-26 16:51:23 -08001548 config_info_line('nonccl', 'Disable NVIDIA NCCL support.')
Austin Anderson6afface2017-12-05 11:59:17 -08001549
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001550
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001551if __name__ == '__main__':
1552 main()