blob: e02428a25a23d66bc26abe9928496cb73b5c3b56 [file] [log] [blame]
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""configure script to get build parameters from user."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
Shanqing Cai71445712018-03-12 19:33:52 -070021import argparse
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070022import errno
23import os
24import platform
25import re
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070026import subprocess
27import sys
28
Andrew Sellec9885ea2017-11-06 09:37:03 -080029# pylint: disable=g-import-not-at-top
Jonathan Hseu008910f2017-08-25 14:01:05 -070030try:
31 from shutil import which
32except ImportError:
33 from distutils.spawn import find_executable as which
Andrew Sellec9885ea2017-11-06 09:37:03 -080034# pylint: enable=g-import-not-at-top
Jonathan Hseu008910f2017-08-25 14:01:05 -070035
A. Unique TensorFlower7ed38302019-11-13 11:14:29 -080036_DEFAULT_CUDA_VERSION = '10.1'
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -070037_DEFAULT_CUDNN_VERSION = '7'
Guangda Laifaa93ac2019-10-23 15:07:12 -070038_DEFAULT_TENSORRT_VERSION = '6'
Smit Hinsufe7d1d92018-07-14 13:16:58 -070039_DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0'
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -070040
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070041_TF_OPENCL_VERSION = '1.2'
42_DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp'
Yifei Fengb1d8c592017-11-22 13:42:21 -080043_DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include'
A. Unique TensorFlower82820ef2018-11-12 13:22:13 -080044_SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15, 16, 17, 18]
Austin Anderson6afface2017-12-05 11:59:17 -080045
46_DEFAULT_PROMPT_ASK_ATTEMPTS = 10
47
Shanqing Cai71445712018-03-12 19:33:52 -070048_TF_BAZELRC_FILENAME = '.tf_configure.bazelrc'
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -070049_TF_WORKSPACE_ROOT = ''
50_TF_BAZELRC = ''
A. Unique TensorFlowered297342019-03-15 11:25:28 -070051_TF_CURRENT_BAZEL_VERSION = None
A. Unique TensorFlower72d0fac2019-11-06 06:24:43 -080052_TF_MIN_BAZEL_VERSION = '0.27.1'
A. Unique TensorFlowera5f9bcd2019-11-12 07:08:54 -080053_TF_MAX_BAZEL_VERSION = '1.1.0'
Shanqing Cai71445712018-03-12 19:33:52 -070054
Jason Furmanek7c234152018-09-26 04:44:12 +000055NCCL_LIB_PATHS = [
TensorFlower Gardenerea5c5292018-10-01 17:28:26 -070056 'lib64/', 'lib/powerpc64le-linux-gnu/', 'lib/x86_64-linux-gnu/', ''
Jason Furmanek7c234152018-09-26 04:44:12 +000057]
Austin Anderson6afface2017-12-05 11:59:17 -080058
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -070059# List of files to configure when building Bazel on Apple platforms.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -080060APPLE_BAZEL_FILES = [
A. Unique TensorFlower6a059c02019-04-05 14:28:27 -070061 'tensorflow/lite/experimental/ios/BUILD',
A. Unique TensorFlower93e70732019-02-14 16:45:32 -080062 'tensorflow/lite/experimental/objc/BUILD',
YoungSeok Yoon7089aaa2019-11-11 21:36:17 -080063 'tensorflow/lite/experimental/swift/BUILD',
64 'tensorflow/lite/tools/benchmark/experimental/ios/BUILD'
A. Unique TensorFlower93e70732019-02-14 16:45:32 -080065]
66
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -070067# List of files to move when building for iOS.
68IOS_FILES = [
69 'tensorflow/lite/experimental/objc/TensorFlowLiteObjC.podspec',
70 'tensorflow/lite/experimental/swift/TensorFlowLiteSwift.podspec',
71]
72
Austin Anderson6afface2017-12-05 11:59:17 -080073
74class UserInputError(Exception):
75 pass
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070076
77
78def is_windows():
79 return platform.system() == 'Windows'
80
81
82def is_linux():
83 return platform.system() == 'Linux'
84
85
86def is_macos():
87 return platform.system() == 'Darwin'
88
89
90def is_ppc64le():
91 return platform.machine() == 'ppc64le'
92
93
Jonathan Hseu008910f2017-08-25 14:01:05 -070094def is_cygwin():
95 return platform.system().startswith('CYGWIN_NT')
96
97
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070098def get_input(question):
99 try:
100 try:
101 answer = raw_input(question)
102 except NameError:
103 answer = input(question) # pylint: disable=bad-builtin
104 except EOFError:
105 answer = ''
106 return answer
107
108
109def symlink_force(target, link_name):
110 """Force symlink, equivalent of 'ln -sf'.
111
112 Args:
113 target: items to link to.
114 link_name: name of the link.
115 """
116 try:
117 os.symlink(target, link_name)
118 except OSError as e:
119 if e.errno == errno.EEXIST:
120 os.remove(link_name)
121 os.symlink(target, link_name)
122 else:
123 raise e
124
125
126def sed_in_place(filename, old, new):
127 """Replace old string with new string in file.
128
129 Args:
130 filename: string for filename.
131 old: string to replace.
132 new: new string to replace to.
133 """
134 with open(filename, 'r') as f:
135 filedata = f.read()
136 newdata = filedata.replace(old, new)
137 with open(filename, 'w') as f:
138 f.write(newdata)
139
140
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700141def write_to_bazelrc(line):
142 with open(_TF_BAZELRC, 'a') as f:
143 f.write(line + '\n')
144
145
146def write_action_env_to_bazelrc(var_name, var):
147 write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var)))
148
149
Jonathan Hseu008910f2017-08-25 14:01:05 -0700150def run_shell(cmd, allow_non_zero=False):
151 if allow_non_zero:
152 try:
153 output = subprocess.check_output(cmd)
154 except subprocess.CalledProcessError as e:
155 output = e.output
156 else:
157 output = subprocess.check_output(cmd)
158 return output.decode('UTF-8').strip()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700159
160
161def cygpath(path):
162 """Convert path from posix to windows."""
Martin Wicked57572e2017-09-02 19:21:45 -0700163 return os.path.abspath(path).replace('\\', '/')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700164
165
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700166def get_python_path(environ_cp, python_bin_path):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700167 """Get the python site package paths."""
168 python_paths = []
169 if environ_cp.get('PYTHONPATH'):
170 python_paths = environ_cp.get('PYTHONPATH').split(':')
171 try:
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700172 library_paths = run_shell([
173 python_bin_path, '-c',
174 'import site; print("\\n".join(site.getsitepackages()))'
175 ]).split('\n')
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700176 except subprocess.CalledProcessError:
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700177 library_paths = [
178 run_shell([
179 python_bin_path, '-c',
180 'from distutils.sysconfig import get_python_lib;'
181 'print(get_python_lib())'
182 ])
183 ]
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700184
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700185 all_paths = set(python_paths + library_paths)
186
187 paths = []
188 for path in all_paths:
189 if os.path.isdir(path):
190 paths.append(path)
191 return paths
192
193
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700194def get_python_major_version(python_bin_path):
195 """Get the python major version."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700196 return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])'])
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700197
198
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -0700199def setup_python(environ_cp):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700200 """Setup python related env variables."""
201 # Get PYTHON_BIN_PATH, default is the current running python.
202 default_python_bin_path = sys.executable
203 ask_python_bin_path = ('Please specify the location of python. [Default is '
204 '%s]: ') % default_python_bin_path
205 while True:
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700206 python_bin_path = get_from_env_or_user_or_default(environ_cp,
207 'PYTHON_BIN_PATH',
208 ask_python_bin_path,
209 default_python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700210 # Check if the path is valid
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700211 if os.path.isfile(python_bin_path) and os.access(python_bin_path, os.X_OK):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700212 break
213 elif not os.path.exists(python_bin_path):
214 print('Invalid python path: %s cannot be found.' % python_bin_path)
215 else:
216 print('%s is not executable. Is it the python binary?' % python_bin_path)
217 environ_cp['PYTHON_BIN_PATH'] = ''
218
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700219 # Convert python path to Windows style before checking lib and version
Martin Wicked57572e2017-09-02 19:21:45 -0700220 if is_windows() or is_cygwin():
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700221 python_bin_path = cygpath(python_bin_path)
222
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700223 # Get PYTHON_LIB_PATH
224 python_lib_path = environ_cp.get('PYTHON_LIB_PATH')
225 if not python_lib_path:
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700226 python_lib_paths = get_python_path(environ_cp, python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700227 if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1':
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700228 python_lib_path = python_lib_paths[0]
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700229 else:
Jonathan Hseu008910f2017-08-25 14:01:05 -0700230 print('Found possible Python library paths:\n %s' %
231 '\n '.join(python_lib_paths))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700232 default_python_lib_path = python_lib_paths[0]
233 python_lib_path = get_input(
Jonathan Hseu008910f2017-08-25 14:01:05 -0700234 'Please input the desired Python library path to use. '
235 'Default is [%s]\n' % python_lib_paths[0])
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700236 if not python_lib_path:
237 python_lib_path = default_python_lib_path
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700238 environ_cp['PYTHON_LIB_PATH'] = python_lib_path
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700239
A. Unique TensorFlowercfccbdb2019-06-24 11:01:58 -0700240 python_major_version = get_python_major_version(python_bin_path)
241 if python_major_version == '2':
242 write_to_bazelrc('build --host_force_python=PY2')
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700243
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700244 # Convert python path to Windows style before writing into bazel.rc
Martin Wicked57572e2017-09-02 19:21:45 -0700245 if is_windows() or is_cygwin():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700246 python_lib_path = cygpath(python_lib_path)
247
248 # Set-up env variables used by python_configure.bzl
249 write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path)
250 write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path)
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -0700251 write_to_bazelrc('build --python_path=\"%s"' % python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700252 environ_cp['PYTHON_BIN_PATH'] = python_bin_path
253
William D. Ironsdcc76a52018-11-20 10:35:18 -0600254 # If choosen python_lib_path is from a path specified in the PYTHONPATH
255 # variable, need to tell bazel to include PYTHONPATH
256 if environ_cp.get('PYTHONPATH'):
257 python_paths = environ_cp.get('PYTHONPATH').split(':')
258 if python_lib_path in python_paths:
TensorFlower Gardener968cd182018-11-28 11:33:16 -0800259 write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH'))
William D. Ironsdcc76a52018-11-20 10:35:18 -0600260
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700261 # Write tools/python_bin_path.sh
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700262 with open(
263 os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'),
264 'w') as f:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700265 f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path)
266
267
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -0700268def reset_tf_configure_bazelrc():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700269 """Reset file that contains customized config settings."""
270 open(_TF_BAZELRC, 'w').close()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700271
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800272
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700273def cleanup_makefile():
274 """Delete any leftover BUILD files from the Makefile build.
275
276 These files could interfere with Bazel parsing.
277 """
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700278 makefile_download_dir = os.path.join(_TF_WORKSPACE_ROOT, 'tensorflow',
279 'contrib', 'makefile', 'downloads')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700280 if os.path.isdir(makefile_download_dir):
281 for root, _, filenames in os.walk(makefile_download_dir):
282 for f in filenames:
283 if f.endswith('BUILD'):
284 os.remove(os.path.join(root, f))
285
286
287def get_var(environ_cp,
288 var_name,
289 query_item,
290 enabled_by_default,
291 question=None,
292 yes_reply=None,
293 no_reply=None):
294 """Get boolean input from user.
295
296 If var_name is not set in env, ask user to enable query_item or not. If the
297 response is empty, use the default.
298
299 Args:
300 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530301 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
302 query_item: string for feature related to the variable, e.g. "CUDA for
303 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700304 enabled_by_default: boolean for default behavior.
305 question: optional string for how to ask for user input.
Michael Cased90054e2018-02-07 14:36:00 -0800306 yes_reply: optional string for reply when feature is enabled.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700307 no_reply: optional string for reply when feature is disabled.
308
309 Returns:
310 boolean value of the variable.
Frank Chenc4ef9272018-01-10 11:36:52 -0800311
312 Raises:
313 UserInputError: if an environment variable is set, but it cannot be
314 interpreted as a boolean indicator, assume that the user has made a
315 scripting error, and will continue to provide invalid input.
316 Raise the error to avoid infinitely looping.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700317 """
318 if not question:
319 question = 'Do you wish to build TensorFlow with %s support?' % query_item
320 if not yes_reply:
321 yes_reply = '%s support will be enabled for TensorFlow.' % query_item
322 if not no_reply:
323 no_reply = 'No %s' % yes_reply
324
325 yes_reply += '\n'
326 no_reply += '\n'
327
328 if enabled_by_default:
329 question += ' [Y/n]: '
330 else:
331 question += ' [y/N]: '
332
333 var = environ_cp.get(var_name)
Frank Chenc4ef9272018-01-10 11:36:52 -0800334 if var is not None:
335 var_content = var.strip().lower()
336 true_strings = ('1', 't', 'true', 'y', 'yes')
337 false_strings = ('0', 'f', 'false', 'n', 'no')
338 if var_content in true_strings:
339 var = True
340 elif var_content in false_strings:
341 var = False
342 else:
343 raise UserInputError(
344 'Environment variable %s must be set as a boolean indicator.\n'
345 'The following are accepted as TRUE : %s.\n'
346 'The following are accepted as FALSE: %s.\n'
A. Unique TensorFlowered297342019-03-15 11:25:28 -0700347 'Current value is %s.' %
348 (var_name, ', '.join(true_strings), ', '.join(false_strings), var))
Frank Chenc4ef9272018-01-10 11:36:52 -0800349
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700350 while var is None:
351 user_input_origin = get_input(question)
352 user_input = user_input_origin.strip().lower()
353 if user_input == 'y':
354 print(yes_reply)
355 var = True
356 elif user_input == 'n':
357 print(no_reply)
358 var = False
359 elif not user_input:
360 if enabled_by_default:
361 print(yes_reply)
362 var = True
363 else:
364 print(no_reply)
365 var = False
366 else:
367 print('Invalid selection: %s' % user_input_origin)
368 return var
369
370
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700371def set_build_var(environ_cp,
372 var_name,
373 query_item,
374 option_name,
375 enabled_by_default,
376 bazel_config_name=None):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700377 """Set if query_item will be enabled for the build.
378
379 Ask user if query_item will be enabled. Default is used if no input is given.
380 Set subprocess environment variable and write to .bazelrc if enabled.
381
382 Args:
383 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530384 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
385 query_item: string for feature related to the variable, e.g. "CUDA for
386 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700387 option_name: string for option to define in .bazelrc.
388 enabled_by_default: boolean for default behavior.
Michael Case98850a52017-09-14 13:35:57 -0700389 bazel_config_name: Name for Bazel --config argument to enable build feature.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700390 """
391
392 var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default)))
393 environ_cp[var_name] = var
394 if var == '1':
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700395 write_to_bazelrc('build:%s --define %s=true' %
396 (bazel_config_name, option_name))
Yifei Fengec451f52018-10-05 12:53:50 -0700397 write_to_bazelrc('build --config=%s' % bazel_config_name)
Michael Case98850a52017-09-14 13:35:57 -0700398 elif bazel_config_name is not None:
399 # TODO(mikecase): Migrate all users of configure.py to use --config Bazel
400 # options and not to set build configs through environment variables.
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700401 write_to_bazelrc('build:%s --define %s=true' %
402 (bazel_config_name, option_name))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700403
404
405def set_action_env_var(environ_cp,
406 var_name,
407 query_item,
408 enabled_by_default,
409 question=None,
410 yes_reply=None,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700411 no_reply=None,
412 bazel_config_name=None):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700413 """Set boolean action_env variable.
414
415 Ask user if query_item will be enabled. Default is used if no input is given.
416 Set environment variable and write to .bazelrc.
417
418 Args:
419 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530420 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
421 query_item: string for feature related to the variable, e.g. "CUDA for
422 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700423 enabled_by_default: boolean for default behavior.
424 question: optional string for how to ask for user input.
Michael Cased90054e2018-02-07 14:36:00 -0800425 yes_reply: optional string for reply when feature is enabled.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700426 no_reply: optional string for reply when feature is disabled.
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700427 bazel_config_name: adding config to .bazelrc instead of action_env.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700428 """
429 var = int(
430 get_var(environ_cp, var_name, query_item, enabled_by_default, question,
431 yes_reply, no_reply))
432
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700433 if not bazel_config_name:
434 write_action_env_to_bazelrc(var_name, var)
435 elif var:
436 write_to_bazelrc('build --config=%s' % bazel_config_name)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700437 environ_cp[var_name] = str(var)
438
439
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700440def convert_version_to_int(version):
441 """Convert a version number to a integer that can be used to compare.
442
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700443 Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The
444 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored.
445
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700446 Args:
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700447 version: a version to be converted
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700448
449 Returns:
450 An integer if converted successfully, otherwise return None.
451 """
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700452 version = version.split('-')[0]
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700453 version_segments = version.split('.')
Austin Anderson87ea41d2019-04-04 10:03:50 -0700454 # Treat "0.24" as "0.24.0"
455 if len(version_segments) == 2:
456 version_segments.append('0')
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700457 for seg in version_segments:
458 if not seg.isdigit():
459 return None
460
461 version_str = ''.join(['%03d' % int(seg) for seg in version_segments])
462 return int(version_str)
463
464
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800465def check_bazel_version(min_version, max_version):
466 """Check installed bazel version is between min_version and max_version.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700467
468 Args:
Mihai Maruseac4db860f2019-04-19 08:52:10 -0700469 min_version: string for minimum bazel version (must exist!).
470 max_version: string for maximum bazel version (must exist!).
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700471
472 Returns:
473 The bazel version detected.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700474 """
Jonathan Hseu008910f2017-08-25 14:01:05 -0700475 if which('bazel') is None:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700476 print('Cannot find bazel. Please install bazel.')
477 sys.exit(0)
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700478 curr_version = run_shell(
479 ['bazel', '--batch', '--bazelrc=/dev/null', 'version'])
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700480
481 for line in curr_version.split('\n'):
482 if 'Build label: ' in line:
483 curr_version = line.split('Build label: ')[1]
484 break
485
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700486 min_version_int = convert_version_to_int(min_version)
487 curr_version_int = convert_version_to_int(curr_version)
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800488 max_version_int = convert_version_to_int(max_version)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700489
490 # Check if current bazel version can be detected properly.
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700491 if not curr_version_int:
492 print('WARNING: current bazel installation is not a release version.')
493 print('Make sure you are running at least bazel %s' % min_version)
494 return curr_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700495
Michael Cased94271a2017-08-22 17:26:52 -0700496 print('You have bazel %s installed.' % curr_version)
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700497
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700498 if curr_version_int < min_version_int:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700499 print('Please upgrade your bazel installation to version %s or higher to '
500 'build TensorFlow!' % min_version)
Jason Zamanb41761c2018-10-14 11:28:53 +0800501 sys.exit(1)
TensorFlower Gardener78c246b2018-12-13 12:37:42 -0800502 if (curr_version_int > max_version_int and
503 'TF_IGNORE_MAX_BAZEL_VERSION' not in os.environ):
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800504 print('Please downgrade your bazel installation to version %s or lower to '
Mihai Maruseace0963c42018-12-20 14:27:40 -0800505 'build TensorFlow! To downgrade: download the installer for the old '
506 'version (from https://github.com/bazelbuild/bazel/releases) then '
507 'run the installer.' % max_version)
Jason Zamanb41761c2018-10-14 11:28:53 +0800508 sys.exit(1)
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700509 return curr_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700510
511
512def set_cc_opt_flags(environ_cp):
513 """Set up architecture-dependent optimization flags.
514
515 Also append CC optimization flags to bazel.rc..
516
517 Args:
518 environ_cp: copy of the os.environ.
519 """
520 if is_ppc64le():
521 # gcc on ppc64le does not support -march, use mcpu instead
522 default_cc_opt_flags = '-mcpu=native'
A. Unique TensorFlower1bba94a2018-04-04 15:45:20 -0700523 elif is_windows():
524 default_cc_opt_flags = '/arch:AVX'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700525 else:
Justin Lebar9ef04f52018-10-10 18:52:45 -0700526 default_cc_opt_flags = '-march=native -Wno-sign-compare'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700527 question = ('Please specify optimization flags to use during compilation when'
528 ' bazel option "--config=opt" is specified [Default is %s]: '
529 ) % default_cc_opt_flags
530 cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS',
531 question, default_cc_opt_flags)
532 for opt in cc_opt_flags.split():
Michael Case00177422017-11-10 13:14:03 -0800533 write_to_bazelrc('build:opt --copt=%s' % opt)
534 # It should be safe on the same build host.
A. Unique TensorFlower1bba94a2018-04-04 15:45:20 -0700535 if not is_ppc64le() and not is_windows():
Shanqing Cai71445712018-03-12 19:33:52 -0700536 write_to_bazelrc('build:opt --host_copt=-march=native')
Michael Casebb3355d2017-11-09 08:46:31 -0800537 write_to_bazelrc('build:opt --define with_default_optimizations=true')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700538
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700539
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700540def set_tf_cuda_clang(environ_cp):
541 """set TF_CUDA_CLANG action_env.
542
543 Args:
544 environ_cp: copy of the os.environ.
545 """
546 question = 'Do you want to use clang as CUDA compiler?'
547 yes_reply = 'Clang will be used as CUDA compiler.'
548 no_reply = 'nvcc will be used as CUDA compiler.'
549 set_action_env_var(
550 environ_cp,
551 'TF_CUDA_CLANG',
552 None,
553 False,
554 question=question,
555 yes_reply=yes_reply,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700556 no_reply=no_reply,
557 bazel_config_name='cuda_clang')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700558
559
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800560def set_tf_download_clang(environ_cp):
561 """Set TF_DOWNLOAD_CLANG action_env."""
Ilya Biryukov9e651e42018-03-22 05:33:42 -0700562 question = 'Do you wish to download a fresh release of clang? (Experimental)'
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800563 yes_reply = 'Clang will be downloaded and used to compile tensorflow.'
564 no_reply = 'Clang will not be downloaded.'
565 set_action_env_var(
566 environ_cp,
567 'TF_DOWNLOAD_CLANG',
568 None,
569 False,
570 question=question,
571 yes_reply=yes_reply,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700572 no_reply=no_reply,
573 bazel_config_name='download_clang')
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800574
575
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700576def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var,
577 var_default):
578 """Get var_name either from env, or user or default.
579
580 If var_name has been set as environment variable, use the preset value, else
581 ask for user input. If no input is provided, the default is used.
582
583 Args:
584 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530585 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700586 ask_for_var: string for how to ask for user input.
587 var_default: default value string.
588
589 Returns:
590 string value for var_name
591 """
592 var = environ_cp.get(var_name)
593 if not var:
594 var = get_input(ask_for_var)
Michael Cased94271a2017-08-22 17:26:52 -0700595 print('\n')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700596 if not var:
597 var = var_default
598 return var
599
600
601def set_clang_cuda_compiler_path(environ_cp):
602 """Set CLANG_CUDA_COMPILER_PATH."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700603 default_clang_path = which('clang') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700604 ask_clang_path = ('Please specify which clang should be used as device and '
605 'host compiler. [Default is %s]: ') % default_clang_path
606
607 while True:
608 clang_cuda_compiler_path = get_from_env_or_user_or_default(
609 environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path,
610 default_clang_path)
611 if os.path.exists(clang_cuda_compiler_path):
612 break
613
614 # Reset and retry
615 print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path)
616 environ_cp['CLANG_CUDA_COMPILER_PATH'] = ''
617
618 # Set CLANG_CUDA_COMPILER_PATH
619 environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path
620 write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH',
621 clang_cuda_compiler_path)
622
623
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700624def prompt_loop_or_load_from_env(environ_cp,
625 var_name,
626 var_default,
627 ask_for_var,
628 check_success,
629 error_msg,
630 suppress_default_error=False,
Austin Anderson58b236b2019-10-22 10:36:10 -0700631 resolve_symlinks=False,
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700632 n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS):
Austin Anderson6afface2017-12-05 11:59:17 -0800633 """Loop over user prompts for an ENV param until receiving a valid response.
634
635 For the env param var_name, read from the environment or verify user input
636 until receiving valid input. When done, set var_name in the environ_cp to its
637 new value.
638
639 Args:
640 environ_cp: (Dict) copy of the os.environ.
641 var_name: (String) string for name of environment variable, e.g. "TF_MYVAR".
642 var_default: (String) default value string.
643 ask_for_var: (String) string for how to ask for user input.
644 check_success: (Function) function that takes one argument and returns a
645 boolean. Should return True if the value provided is considered valid. May
646 contain a complex error message if error_msg does not provide enough
647 information. In that case, set suppress_default_error to True.
648 error_msg: (String) String with one and only one '%s'. Formatted with each
649 invalid response upon check_success(input) failure.
650 suppress_default_error: (Bool) Suppress the above error message in favor of
651 one from the check_success function.
Austin Anderson58b236b2019-10-22 10:36:10 -0700652 resolve_symlinks: (Bool) Translate symbolic links into the real filepath.
Austin Anderson6afface2017-12-05 11:59:17 -0800653 n_ask_attempts: (Integer) Number of times to query for valid input before
654 raising an error and quitting.
655
656 Returns:
657 [String] The value of var_name after querying for input.
658
659 Raises:
660 UserInputError: if a query has been attempted n_ask_attempts times without
Frank Chenc4ef9272018-01-10 11:36:52 -0800661 success, assume that the user has made a scripting error, and will
662 continue to provide invalid input. Raise the error to avoid infinitely
663 looping.
Austin Anderson6afface2017-12-05 11:59:17 -0800664 """
665 default = environ_cp.get(var_name) or var_default
666 full_query = '%s [Default is %s]: ' % (
667 ask_for_var,
668 default,
669 )
670
671 for _ in range(n_ask_attempts):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700672 val = get_from_env_or_user_or_default(environ_cp, var_name, full_query,
Austin Anderson6afface2017-12-05 11:59:17 -0800673 default)
674 if check_success(val):
675 break
676 if not suppress_default_error:
677 print(error_msg % val)
678 environ_cp[var_name] = ''
679 else:
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700680 raise UserInputError('Invalid %s setting was provided %d times in a row. '
681 'Assuming to be a scripting mistake.' %
682 (var_name, n_ask_attempts))
Austin Anderson6afface2017-12-05 11:59:17 -0800683
Austin Anderson58b236b2019-10-22 10:36:10 -0700684 if resolve_symlinks and os.path.islink(val):
685 val = os.path.realpath(val)
Austin Anderson6afface2017-12-05 11:59:17 -0800686 environ_cp[var_name] = val
687 return val
688
689
690def create_android_ndk_rule(environ_cp):
691 """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule."""
692 if is_windows() or is_cygwin():
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700693 default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' %
694 environ_cp['APPDATA'])
Austin Anderson6afface2017-12-05 11:59:17 -0800695 elif is_macos():
696 default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME']
697 else:
698 default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME']
699
700 def valid_ndk_path(path):
701 return (os.path.exists(path) and
702 os.path.exists(os.path.join(path, 'source.properties')))
703
704 android_ndk_home_path = prompt_loop_or_load_from_env(
705 environ_cp,
706 var_name='ANDROID_NDK_HOME',
707 var_default=default_ndk_path,
708 ask_for_var='Please specify the home path of the Android NDK to use.',
709 check_success=valid_ndk_path,
710 error_msg=('The path %s or its child file "source.properties" '
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700711 'does not exist.'))
Michael Case51053502018-06-05 17:47:19 -0700712 write_action_env_to_bazelrc('ANDROID_NDK_HOME', android_ndk_home_path)
Jared Dukea0104b72019-04-04 12:23:58 -0700713 write_action_env_to_bazelrc(
714 'ANDROID_NDK_API_LEVEL',
715 get_ndk_api_level(environ_cp, android_ndk_home_path))
Austin Anderson6afface2017-12-05 11:59:17 -0800716
717
718def create_android_sdk_rule(environ_cp):
719 """Set Android variables and write Android SDK WORKSPACE rule."""
720 if is_windows() or is_cygwin():
721 default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA'])
722 elif is_macos():
Shashi Shekharc0ff0cc2018-07-17 09:00:24 -0700723 default_sdk_path = '%s/library/Android/Sdk' % environ_cp['HOME']
Austin Anderson6afface2017-12-05 11:59:17 -0800724 else:
725 default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME']
726
727 def valid_sdk_path(path):
728 return (os.path.exists(path) and
729 os.path.exists(os.path.join(path, 'platforms')) and
730 os.path.exists(os.path.join(path, 'build-tools')))
731
732 android_sdk_home_path = prompt_loop_or_load_from_env(
733 environ_cp,
734 var_name='ANDROID_SDK_HOME',
735 var_default=default_sdk_path,
736 ask_for_var='Please specify the home path of the Android SDK to use.',
737 check_success=valid_sdk_path,
738 error_msg=('Either %s does not exist, or it does not contain the '
739 'subdirectories "platforms" and "build-tools".'))
740
741 platforms = os.path.join(android_sdk_home_path, 'platforms')
742 api_levels = sorted(os.listdir(platforms))
743 api_levels = [x.replace('android-', '') for x in api_levels]
744
745 def valid_api_level(api_level):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700746 return os.path.exists(
747 os.path.join(android_sdk_home_path, 'platforms',
748 'android-' + api_level))
Austin Anderson6afface2017-12-05 11:59:17 -0800749
750 android_api_level = prompt_loop_or_load_from_env(
751 environ_cp,
752 var_name='ANDROID_API_LEVEL',
753 var_default=api_levels[-1],
754 ask_for_var=('Please specify the Android SDK API level to use. '
755 '[Available levels: %s]') % api_levels,
756 check_success=valid_api_level,
757 error_msg='Android-%s is not present in the SDK path.')
758
759 build_tools = os.path.join(android_sdk_home_path, 'build-tools')
760 versions = sorted(os.listdir(build_tools))
761
762 def valid_build_tools(version):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700763 return os.path.exists(
764 os.path.join(android_sdk_home_path, 'build-tools', version))
Austin Anderson6afface2017-12-05 11:59:17 -0800765
766 android_build_tools_version = prompt_loop_or_load_from_env(
767 environ_cp,
768 var_name='ANDROID_BUILD_TOOLS_VERSION',
769 var_default=versions[-1],
770 ask_for_var=('Please specify an Android build tools version to use. '
771 '[Available versions: %s]') % versions,
772 check_success=valid_build_tools,
773 error_msg=('The selected SDK does not have build-tools version %s '
774 'available.'))
775
Michael Case51053502018-06-05 17:47:19 -0700776 write_action_env_to_bazelrc('ANDROID_BUILD_TOOLS_VERSION',
777 android_build_tools_version)
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700778 write_action_env_to_bazelrc('ANDROID_SDK_API_LEVEL', android_api_level)
779 write_action_env_to_bazelrc('ANDROID_SDK_HOME', android_sdk_home_path)
Austin Anderson6afface2017-12-05 11:59:17 -0800780
781
Jared Dukea0104b72019-04-04 12:23:58 -0700782def get_ndk_api_level(environ_cp, android_ndk_home_path):
783 """Gets the appropriate NDK API level to use for the provided Android NDK path."""
784
785 # First check to see if we're using a blessed version of the NDK.
Austin Anderson6afface2017-12-05 11:59:17 -0800786 properties_path = '%s/source.properties' % android_ndk_home_path
787 if is_windows() or is_cygwin():
788 properties_path = cygpath(properties_path)
789 with open(properties_path, 'r') as f:
790 filedata = f.read()
791
792 revision = re.search(r'Pkg.Revision = (\d+)', filedata)
793 if revision:
Jared Dukea0104b72019-04-04 12:23:58 -0700794 ndk_version = revision.group(1)
Michael Case51053502018-06-05 17:47:19 -0700795 else:
796 raise Exception('Unable to parse NDK revision.')
Jared Dukea0104b72019-04-04 12:23:58 -0700797 if int(ndk_version) not in _SUPPORTED_ANDROID_NDK_VERSIONS:
798 print('WARNING: The NDK version in %s is %s, which is not '
799 'supported by Bazel (officially supported versions: %s). Please use '
800 'another version. Compiling Android targets may result in confusing '
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700801 'errors.\n' %
802 (android_ndk_home_path, ndk_version, _SUPPORTED_ANDROID_NDK_VERSIONS))
Jared Dukea0104b72019-04-04 12:23:58 -0700803
804 # Now grab the NDK API level to use. Note that this is different from the
805 # SDK API level, as the NDK API level is effectively the *min* target SDK
806 # version.
807 platforms = os.path.join(android_ndk_home_path, 'platforms')
808 api_levels = sorted(os.listdir(platforms))
809 api_levels = [
810 x.replace('android-', '') for x in api_levels if 'android-' in x
811 ]
812
813 def valid_api_level(api_level):
814 return os.path.exists(
815 os.path.join(android_ndk_home_path, 'platforms',
816 'android-' + api_level))
817
818 android_ndk_api_level = prompt_loop_or_load_from_env(
819 environ_cp,
820 var_name='ANDROID_NDK_API_LEVEL',
Jared Dukea1309f42019-11-15 17:09:35 -0800821 var_default='21', # 21 is required for ARM64 support.
Jared Dukea0104b72019-04-04 12:23:58 -0700822 ask_for_var=('Please specify the (min) Android NDK API level to use. '
823 '[Available levels: %s]') % api_levels,
824 check_success=valid_api_level,
825 error_msg='Android-%s is not present in the NDK path.')
826
827 return android_ndk_api_level
Austin Anderson6afface2017-12-05 11:59:17 -0800828
829
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700830def set_gcc_host_compiler_path(environ_cp):
831 """Set GCC_HOST_COMPILER_PATH."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700832 default_gcc_host_compiler_path = which('gcc') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700833 cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH')
834
835 if os.path.islink(cuda_bin_symlink):
836 # os.readlink is only available in linux
Jonathan Hseu008910f2017-08-25 14:01:05 -0700837 default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700838
Austin Anderson6afface2017-12-05 11:59:17 -0800839 gcc_host_compiler_path = prompt_loop_or_load_from_env(
840 environ_cp,
841 var_name='GCC_HOST_COMPILER_PATH',
842 var_default=default_gcc_host_compiler_path,
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800843 ask_for_var='Please specify which gcc should be used by nvcc as the host compiler.',
Austin Anderson6afface2017-12-05 11:59:17 -0800844 check_success=os.path.exists,
Austin Anderson58b236b2019-10-22 10:36:10 -0700845 resolve_symlinks=True,
Austin Anderson6afface2017-12-05 11:59:17 -0800846 error_msg='Invalid gcc path. %s cannot be found.',
847 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700848
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700849 write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path)
850
851
Ankur Taly0e6f39d2018-02-16 18:22:55 -0800852def reformat_version_sequence(version_str, sequence_count):
853 """Reformat the version string to have the given number of sequences.
854
855 For example:
856 Given (7, 2) -> 7.0
857 (7.0.1, 2) -> 7.0
858 (5, 1) -> 5
859 (5.0.3.2, 1) -> 5
860
861 Args:
862 version_str: String, the version string.
863 sequence_count: int, an integer.
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700864
Ankur Taly0e6f39d2018-02-16 18:22:55 -0800865 Returns:
866 string, reformatted version string.
867 """
868 v = version_str.split('.')
869 if len(v) < sequence_count:
870 v = v + (['0'] * (sequence_count - len(v)))
871
872 return '.'.join(v[:sequence_count])
873
874
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700875def set_tf_cuda_paths(environ_cp):
876 """Set TF_CUDA_PATHS."""
877 ask_cuda_paths = (
878 'Please specify the comma-separated list of base paths to look for CUDA '
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700879 'libraries and headers. [Leave empty to use the default]: ')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700880 tf_cuda_paths = get_from_env_or_user_or_default(environ_cp, 'TF_CUDA_PATHS',
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700881 ask_cuda_paths, '')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700882 if tf_cuda_paths:
883 environ_cp['TF_CUDA_PATHS'] = tf_cuda_paths
884
885
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700886def set_tf_cuda_version(environ_cp):
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700887 """Set TF_CUDA_VERSION."""
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700888 ask_cuda_version = (
889 'Please specify the CUDA SDK version you want to use. '
890 '[Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700891 tf_cuda_version = get_from_env_or_user_or_default(environ_cp,
892 'TF_CUDA_VERSION',
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700893 ask_cuda_version,
894 _DEFAULT_CUDA_VERSION)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700895 environ_cp['TF_CUDA_VERSION'] = tf_cuda_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700896
897
Yifei Fengb1d8c592017-11-22 13:42:21 -0800898def set_tf_cudnn_version(environ_cp):
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700899 """Set TF_CUDNN_VERSION."""
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700900 ask_cudnn_version = (
901 'Please specify the cuDNN version you want to use. '
902 '[Leave empty to default to cuDNN %s]: ') % _DEFAULT_CUDNN_VERSION
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700903 tf_cudnn_version = get_from_env_or_user_or_default(environ_cp,
904 'TF_CUDNN_VERSION',
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700905 ask_cudnn_version,
906 _DEFAULT_CUDNN_VERSION)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700907 environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700908
909
Mingxing Tan1e7b0e42018-06-28 19:13:20 -0700910def is_cuda_compatible(lib, cuda_ver, cudnn_ver):
911 """Check compatibility between given library and cudnn/cudart libraries."""
912 ldd_bin = which('ldd') or '/usr/bin/ldd'
913 ldd_out = run_shell([ldd_bin, lib], True)
914 ldd_out = ldd_out.split(os.linesep)
915 cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$')
916 cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$')
917 cudnn = None
918 cudart = None
919 cudnn_ok = True # assume no cudnn dependency by default
920 cuda_ok = True # assume no cuda dependency by default
921 for line in ldd_out:
922 if 'libcudnn.so' in line:
923 cudnn = cudnn_pattern.search(line)
924 cudnn_ok = False
925 elif 'libcudart.so' in line:
926 cudart = cuda_pattern.search(line)
927 cuda_ok = False
928 if cudnn and len(cudnn.group(1)):
929 cudnn = convert_version_to_int(cudnn.group(1))
930 if cudart and len(cudart.group(1)):
931 cudart = convert_version_to_int(cudart.group(1))
932 if cudnn is not None:
933 cudnn_ok = (cudnn == cudnn_ver)
934 if cudart is not None:
935 cuda_ok = (cudart == cuda_ver)
936 return cudnn_ok and cuda_ok
937
938
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700939def set_tf_tensorrt_version(environ_cp):
940 """Set TF_TENSORRT_VERSION."""
Guangda Lai76f69382018-01-25 23:59:19 -0800941 if not is_linux():
942 raise ValueError('Currently TensorRT is only supported on Linux platform.')
943
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700944 if not int(environ_cp.get('TF_NEED_TENSORRT', False)):
Guangda Lai76f69382018-01-25 23:59:19 -0800945 return
946
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700947 ask_tensorrt_version = (
948 'Please specify the TensorRT version you want to use. '
Guangda Laifaa93ac2019-10-23 15:07:12 -0700949 '[Leave empty to default to TensorRT %s]: ') % _DEFAULT_TENSORRT_VERSION
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700950 tf_tensorrt_version = get_from_env_or_user_or_default(
951 environ_cp, 'TF_TENSORRT_VERSION', ask_tensorrt_version,
952 _DEFAULT_TENSORRT_VERSION)
Guangda Lai76f69382018-01-25 23:59:19 -0800953 environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version
Guangda Lai76f69382018-01-25 23:59:19 -0800954
955
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700956def set_tf_nccl_version(environ_cp):
957 """Set TF_NCCL_VERSION."""
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700958 if not is_linux():
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700959 raise ValueError('Currently NCCL is only supported on Linux platform.')
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700960
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700961 if 'TF_NCCL_VERSION' in environ_cp:
962 return
963
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700964 ask_nccl_version = (
A. Unique TensorFlower53faa312018-10-05 08:46:54 -0700965 'Please specify the locally installed NCCL version you want to use. '
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700966 '[Leave empty to use http://github.com/nvidia/nccl]: ')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700967 tf_nccl_version = get_from_env_or_user_or_default(environ_cp,
968 'TF_NCCL_VERSION',
969 ask_nccl_version, '')
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700970 environ_cp['TF_NCCL_VERSION'] = tf_nccl_version
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800971
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700972
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700973def get_native_cuda_compute_capabilities(environ_cp):
974 """Get native cuda compute capabilities.
975
976 Args:
977 environ_cp: copy of the os.environ.
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700978
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700979 Returns:
980 string of native cuda compute capabilities, separated by comma.
981 """
982 device_query_bin = os.path.join(
983 environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery')
Jonathan Hseu008910f2017-08-25 14:01:05 -0700984 if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK):
985 try:
986 output = run_shell(device_query_bin).split('\n')
987 pattern = re.compile('[0-9]*\\.[0-9]*')
988 output = [pattern.search(x) for x in output if 'Capability' in x]
989 output = ','.join(x.group() for x in output if x is not None)
990 except subprocess.CalledProcessError:
991 output = ''
992 else:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700993 output = ''
994 return output
995
996
997def set_tf_cuda_compute_capabilities(environ_cp):
998 """Set TF_CUDA_COMPUTE_CAPABILITIES."""
999 while True:
1000 native_cuda_compute_capabilities = get_native_cuda_compute_capabilities(
1001 environ_cp)
1002 if not native_cuda_compute_capabilities:
1003 default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES
1004 else:
1005 default_cuda_compute_capabilities = native_cuda_compute_capabilities
1006
1007 ask_cuda_compute_capabilities = (
1008 'Please specify a list of comma-separated '
P Sudeepam52093562019-02-17 17:34:01 +05301009 'CUDA compute capabilities you want to '
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001010 'build with.\nYou can find the compute '
1011 'capability of your device at: '
1012 'https://developer.nvidia.com/cuda-gpus.\nPlease'
1013 ' note that each additional compute '
1014 'capability significantly increases your '
P Sudeepam52093562019-02-17 17:34:01 +05301015 'build time and binary size, and that '
1016 'TensorFlow only supports compute '
P Sudeepam765ceda2019-02-17 17:39:08 +05301017 'capabilities >= 3.5 [Default is: %s]: ' %
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001018 default_cuda_compute_capabilities)
1019 tf_cuda_compute_capabilities = get_from_env_or_user_or_default(
1020 environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES',
1021 ask_cuda_compute_capabilities, default_cuda_compute_capabilities)
1022 # Check whether all capabilities from the input is valid
1023 all_valid = True
Maciejd0f5bc12018-04-30 22:30:58 -05001024 # Remove all whitespace characters before splitting the string
Michael Case51053502018-06-05 17:47:19 -07001025 # that users may insert by accident, as this will result in error
Maciejd0f5bc12018-04-30 22:30:58 -05001026 tf_cuda_compute_capabilities = ''.join(tf_cuda_compute_capabilities.split())
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001027 for compute_capability in tf_cuda_compute_capabilities.split(','):
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001028 m = re.match('[0-9]+.[0-9]+', compute_capability)
1029 if not m:
Austin Anderson32202dc2019-02-19 10:46:27 -08001030 print('Invalid compute capability: %s' % compute_capability)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001031 all_valid = False
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001032 else:
P Sudeepam52093562019-02-17 17:34:01 +05301033 ver = float(m.group(0))
A. Unique TensorFlower8dc2d0e2019-03-12 01:41:05 -07001034 if ver < 3.0:
1035 print('ERROR: TensorFlow only supports CUDA compute capabilities 3.0 '
Austin Anderson32202dc2019-02-19 10:46:27 -08001036 'and higher. Please re-specify the list of compute '
1037 'capabilities excluding version %s.' % ver)
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001038 all_valid = False
A. Unique TensorFlower8dc2d0e2019-03-12 01:41:05 -07001039 if ver < 3.5:
1040 print('WARNING: XLA does not support CUDA compute capabilities '
1041 'lower than 3.5. Disable XLA when running on older GPUs.')
P Sudeepam765ceda2019-02-17 17:39:08 +05301042
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001043 if all_valid:
1044 break
1045
1046 # Reset and Retry
1047 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = ''
1048
1049 # Set TF_CUDA_COMPUTE_CAPABILITIES
1050 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities
1051 write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES',
1052 tf_cuda_compute_capabilities)
1053
1054
1055def set_other_cuda_vars(environ_cp):
1056 """Set other CUDA related variables."""
A. Unique TensorFlowerab391982018-07-11 04:52:49 -07001057 # If CUDA is enabled, always use GPU during build and test.
1058 if environ_cp.get('TF_CUDA_CLANG') == '1':
1059 write_to_bazelrc('build --config=cuda_clang')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001060 else:
A. Unique TensorFlowerab391982018-07-11 04:52:49 -07001061 write_to_bazelrc('build --config=cuda')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001062
1063
1064def set_host_cxx_compiler(environ_cp):
1065 """Set HOST_CXX_COMPILER."""
Jonathan Hseu008910f2017-08-25 14:01:05 -07001066 default_cxx_host_compiler = which('g++') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001067
Austin Anderson6afface2017-12-05 11:59:17 -08001068 host_cxx_compiler = prompt_loop_or_load_from_env(
1069 environ_cp,
1070 var_name='HOST_CXX_COMPILER',
1071 var_default=default_cxx_host_compiler,
1072 ask_for_var=('Please specify which C++ compiler should be used as the '
1073 'host C++ compiler.'),
1074 check_success=os.path.exists,
1075 error_msg='Invalid C++ compiler path. %s cannot be found.',
1076 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001077
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001078 write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler)
1079
1080
1081def set_host_c_compiler(environ_cp):
1082 """Set HOST_C_COMPILER."""
Jonathan Hseu008910f2017-08-25 14:01:05 -07001083 default_c_host_compiler = which('gcc') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001084
Austin Anderson6afface2017-12-05 11:59:17 -08001085 host_c_compiler = prompt_loop_or_load_from_env(
1086 environ_cp,
1087 var_name='HOST_C_COMPILER',
1088 var_default=default_c_host_compiler,
Shanqing Cai71445712018-03-12 19:33:52 -07001089 ask_for_var=('Please specify which C compiler should be used as the host '
Austin Anderson6afface2017-12-05 11:59:17 -08001090 'C compiler.'),
1091 check_success=os.path.exists,
1092 error_msg='Invalid C compiler path. %s cannot be found.',
1093 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001094
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001095 write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler)
1096
1097
1098def set_computecpp_toolkit_path(environ_cp):
1099 """Set COMPUTECPP_TOOLKIT_PATH."""
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001100
Austin Anderson6afface2017-12-05 11:59:17 -08001101 def toolkit_exists(toolkit_path):
1102 """Check if a computecpp toolkit path is valid."""
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001103 if is_linux():
1104 sycl_rt_lib_path = 'lib/libComputeCpp.so'
1105 else:
1106 sycl_rt_lib_path = ''
1107
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001108 sycl_rt_lib_path_full = os.path.join(toolkit_path, sycl_rt_lib_path)
Austin Anderson6afface2017-12-05 11:59:17 -08001109 exists = os.path.exists(sycl_rt_lib_path_full)
1110 if not exists:
1111 print('Invalid SYCL %s library path. %s cannot be found' %
1112 (_TF_OPENCL_VERSION, sycl_rt_lib_path_full))
1113 return exists
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001114
Austin Anderson6afface2017-12-05 11:59:17 -08001115 computecpp_toolkit_path = prompt_loop_or_load_from_env(
1116 environ_cp,
1117 var_name='COMPUTECPP_TOOLKIT_PATH',
1118 var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH,
1119 ask_for_var=(
1120 'Please specify the location where ComputeCpp for SYCL %s is '
1121 'installed.' % _TF_OPENCL_VERSION),
1122 check_success=toolkit_exists,
1123 error_msg='Invalid SYCL compiler path. %s cannot be found.',
1124 suppress_default_error=True)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001125
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001126 write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH',
1127 computecpp_toolkit_path)
1128
Michael Cased31531a2018-01-05 14:09:41 -08001129
Dandelion Man?90e42f32017-12-15 18:15:07 -08001130def set_trisycl_include_dir(environ_cp):
Michael Cased31531a2018-01-05 14:09:41 -08001131 """Set TRISYCL_INCLUDE_DIR."""
Frank Chenc4ef9272018-01-10 11:36:52 -08001132
Dandelion Man?90e42f32017-12-15 18:15:07 -08001133 ask_trisycl_include_dir = ('Please specify the location of the triSYCL '
1134 'include directory. (Use --config=sycl_trisycl '
1135 'when building with Bazel) '
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001136 '[Default is %s]: ') % (
1137 _DEFAULT_TRISYCL_INCLUDE_DIR)
Frank Chenc4ef9272018-01-10 11:36:52 -08001138
Dandelion Man?90e42f32017-12-15 18:15:07 -08001139 while True:
1140 trisycl_include_dir = get_from_env_or_user_or_default(
Michael Cased31531a2018-01-05 14:09:41 -08001141 environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir,
1142 _DEFAULT_TRISYCL_INCLUDE_DIR)
Dandelion Man?90e42f32017-12-15 18:15:07 -08001143 if os.path.exists(trisycl_include_dir):
1144 break
1145
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001146 print('Invalid triSYCL include directory, %s cannot be found' %
1147 (trisycl_include_dir))
Dandelion Man?90e42f32017-12-15 18:15:07 -08001148
1149 # Set TRISYCL_INCLUDE_DIR
1150 environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001151 write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', trisycl_include_dir)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001152
Yifei Fengb1d8c592017-11-22 13:42:21 -08001153
A. Unique TensorFlowere9797fd2019-03-22 11:02:40 -07001154def system_specific_test_config(env):
A. Unique TensorFlower7bd86372019-03-21 15:19:30 -07001155 """Add default build and test flags required for TF tests to bazelrc."""
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001156 write_to_bazelrc('test --flaky_test_attempts=3')
1157 write_to_bazelrc('test --test_size_filters=small,medium')
frreiss8b013a72019-11-20 11:36:27 -08001158
1159 # Each instance of --test_tag_filters or --build_tag_filters overrides all
1160 # previous instances, so we need to build up a complete list and write a
1161 # single list of filters for the .bazelrc file.
1162
1163 # Filters to use with both --test_tag_filters and --build_tag_filters
frreissbab7d7d2019-11-21 10:52:55 -08001164 test_and_build_filters = ['-benchmark-test', '-no_oss']
1165 # Additional filters for --test_tag_filters beyond those in
1166 # test_and_build_filters
1167 test_only_filters = ['-oss_serial']
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001168 if is_windows():
frreissbab7d7d2019-11-21 10:52:55 -08001169 test_and_build_filters.append('-no_windows')
Guangda Laibcd701a2019-03-12 21:04:51 -07001170 if env.get('TF_NEED_CUDA', None) == '1':
frreissbab7d7d2019-11-21 10:52:55 -08001171 test_and_build_filters += ['-no_windows_gpu', '-no_gpu']
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001172 else:
frreissbab7d7d2019-11-21 10:52:55 -08001173 test_and_build_filters.append('-gpu')
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001174 elif is_macos():
frreissbab7d7d2019-11-21 10:52:55 -08001175 test_and_build_filters += ['-gpu', '-nomac', '-no_mac']
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001176 elif is_linux():
Guangda Laibcd701a2019-03-12 21:04:51 -07001177 if env.get('TF_NEED_CUDA', None) == '1':
frreissbab7d7d2019-11-21 10:52:55 -08001178 test_and_build_filters.append('-no_gpu')
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001179 write_to_bazelrc('test --test_env=LD_LIBRARY_PATH')
1180 else:
frreissbab7d7d2019-11-21 10:52:55 -08001181 test_and_build_filters.append('-gpu')
TensorFlower Gardenere5159082019-11-22 11:47:38 -08001182 write_to_bazelrc('test --test_tag_filters=%s' %
1183 ','.join(test_and_build_filters + test_only_filters))
1184 write_to_bazelrc('test --build_tag_filters=%s' %
1185 ','.join(test_and_build_filters))
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001186
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001187
Yifei Feng5198cb82018-08-17 13:53:06 -07001188def set_system_libs_flag(environ_cp):
1189 syslibs = environ_cp.get('TF_SYSTEM_LIBS', '')
TensorFlower Gardener61a87202018-10-01 12:25:39 -07001190 if syslibs:
Jason Zaman5fc39bd2018-09-16 01:38:55 +08001191 if ',' in syslibs:
1192 syslibs = ','.join(sorted(syslibs.split(',')))
1193 else:
1194 syslibs = ','.join(sorted(syslibs.split()))
Yifei Feng5198cb82018-08-17 13:53:06 -07001195 write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs)
1196
Jason Zaman5fc39bd2018-09-16 01:38:55 +08001197 if 'PREFIX' in environ_cp:
1198 write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX'])
1199 if 'LIBDIR' in environ_cp:
1200 write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR'])
1201 if 'INCLUDEDIR' in environ_cp:
1202 write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR'])
1203
Yifei Feng5198cb82018-08-17 13:53:06 -07001204
Austin Anderson6d7c3762019-11-19 17:04:19 -08001205def is_reduced_optimize_huge_functions_available(environ_cp):
1206 """Check to see if the system supports /d2ReducedOptimizeHugeFunctions.
1207
1208 The above compiler flag is a new compiler flag introduced to the Visual Studio
1209 compiler in version 16.4 (available in Visual Studio 2019, Preview edition
1210 only, as of 2019-11-19). TensorFlow needs this flag to massively reduce
1211 compile times, but until 16.4 is officially released, we can't depend on it.
1212
1213 See also https://groups.google.com/a/tensorflow.org/g/build/c/SsW98Eo7l3o
1214
1215 Because it's very annoying to check this manually (to check the MSVC installed
1216 versions, you need to use the registry, and it's not clear if Bazel will be
1217 using that install version anyway), we expect enviroments who know they may
1218 use this flag to export TF_VC_VERSION=16.4
1219
1220 TODO(angerson, gunan): Remove this function when TensorFlow's minimum VS
1221 version is upgraded to 16.4.
1222
1223 Arguments:
1224 environ_cp: Environment of the current execution
1225
1226 Returns:
1227 boolean, whether or not /d2ReducedOptimizeHugeFunctions is available on this
1228 machine.
1229 """
1230 return float(environ_cp.get('TF_VC_VERSION', '0')) >= 16.4
1231
1232
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001233def set_windows_build_flags(environ_cp):
1234 """Set Windows specific build options."""
1235 # The non-monolithic build is not supported yet
1236 write_to_bazelrc('build --config monolithic')
1237 # Suppress warning messages
1238 write_to_bazelrc('build --copt=-w --host_copt=-w')
Loo Rong Jie31f10d22019-02-02 10:03:20 +08001239 # Fix winsock2.h conflicts
TensorFlower Gardener345cccf2019-02-28 15:22:59 -08001240 write_to_bazelrc(
Justin Lebar7b57c5a2019-04-30 12:24:29 -07001241 'build --copt=-DWIN32_LEAN_AND_MEAN --host_copt=-DWIN32_LEAN_AND_MEAN '
1242 '--copt=-DNOGDI --host_copt=-DNOGDI')
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001243 # Output more verbose information when something goes wrong
1244 write_to_bazelrc('build --verbose_failures')
1245 # The host and target platforms are the same in Windows build. So we don't
1246 # have to distinct them. This avoids building the same targets twice.
1247 write_to_bazelrc('build --distinct_host_configuration=false')
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001248
Austin Anderson6d7c3762019-11-19 17:04:19 -08001249 if is_reduced_optimize_huge_functions_available(environ_cp):
1250 write_to_bazelrc(
1251 'build --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions'
1252 )
1253
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001254 if get_var(
1255 environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline',
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001256 True, ('Would you like to override eigen strong inline for some C++ '
1257 'compilation to reduce the compilation time?'),
1258 'Eigen strong inline overridden.', 'Not overriding eigen strong inline, '
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001259 'some compilations could take more than 20 mins.'):
1260 # Due to a known MSVC compiler issue
1261 # https://github.com/tensorflow/tensorflow/issues/10521
1262 # Overriding eigen strong inline speeds up the compiling of
1263 # conv_grad_ops_3d.cc and conv_ops_3d.cc by 20 minutes,
1264 # but this also hurts the performance. Let users decide what they want.
1265 write_to_bazelrc('build --define=override_eigen_strong_inline=true')
Dandelion Man?90e42f32017-12-15 18:15:07 -08001266
A. Unique TensorFlower061c3592017-11-13 14:21:04 -08001267
Michael Cased31531a2018-01-05 14:09:41 -08001268def config_info_line(name, help_text):
1269 """Helper function to print formatted help text for Bazel config options."""
1270 print('\t--config=%-12s\t# %s' % (name, help_text))
1271
1272
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001273def configure_ios():
1274 """Configures TensorFlow for iOS builds.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001275
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001276 This function will only be executed if `is_macos()` is true.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001277 """
1278 if not is_macos():
1279 return
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001280 for filepath in APPLE_BAZEL_FILES:
1281 existing_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath + '.apple')
1282 renamed_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath)
1283 symlink_force(existing_filepath, renamed_filepath)
1284 for filepath in IOS_FILES:
1285 filename = os.path.basename(filepath)
1286 new_filepath = os.path.join(_TF_WORKSPACE_ROOT, filename)
1287 symlink_force(filepath, new_filepath)
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001288
1289
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001290def validate_cuda_config(environ_cp):
1291 """Run find_cuda_config.py and return cuda_toolkit_path, or None."""
1292
1293 def maybe_encode_env(env):
1294 """Encodes unicode in env to str on Windows python 2.x."""
1295 if not is_windows() or sys.version_info[0] != 2:
1296 return env
1297 for k, v in env.items():
1298 if isinstance(k, unicode):
1299 k = k.encode('ascii')
1300 if isinstance(v, unicode):
1301 v = v.encode('ascii')
1302 env[k] = v
1303 return env
1304
1305 cuda_libraries = ['cuda', 'cudnn']
1306 if is_linux():
A. Unique TensorFlower28fc9cc2019-05-01 14:17:54 -07001307 if int(environ_cp.get('TF_NEED_TENSORRT', False)):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001308 cuda_libraries.append('tensorrt')
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001309 if environ_cp.get('TF_NCCL_VERSION', None):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001310 cuda_libraries.append('nccl')
1311
1312 proc = subprocess.Popen(
1313 [environ_cp['PYTHON_BIN_PATH'], 'third_party/gpus/find_cuda_config.py'] +
1314 cuda_libraries,
1315 stdout=subprocess.PIPE,
1316 env=maybe_encode_env(environ_cp))
1317
1318 if proc.wait():
1319 # Errors from find_cuda_config.py were sent to stderr.
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -07001320 print('Asking for detailed CUDA configuration...\n')
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001321 return False
1322
1323 config = dict(
1324 tuple(line.decode('ascii').rstrip().split(': ')) for line in proc.stdout)
1325
1326 print('Found CUDA %s in:' % config['cuda_version'])
1327 print(' %s' % config['cuda_library_dir'])
1328 print(' %s' % config['cuda_include_dir'])
1329
1330 print('Found cuDNN %s in:' % config['cudnn_version'])
1331 print(' %s' % config['cudnn_library_dir'])
1332 print(' %s' % config['cudnn_include_dir'])
1333
1334 if 'tensorrt_version' in config:
1335 print('Found TensorRT %s in:' % config['tensorrt_version'])
1336 print(' %s' % config['tensorrt_library_dir'])
1337 print(' %s' % config['tensorrt_include_dir'])
1338
1339 if config.get('nccl_version', None):
1340 print('Found NCCL %s in:' % config['nccl_version'])
1341 print(' %s' % config['nccl_library_dir'])
1342 print(' %s' % config['nccl_include_dir'])
1343
1344 print('\n')
1345
1346 environ_cp['CUDA_TOOLKIT_PATH'] = config['cuda_toolkit_path']
1347 return True
1348
1349
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001350def main():
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001351 global _TF_WORKSPACE_ROOT
1352 global _TF_BAZELRC
A. Unique TensorFlowered297342019-03-15 11:25:28 -07001353 global _TF_CURRENT_BAZEL_VERSION
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001354
Shanqing Cai71445712018-03-12 19:33:52 -07001355 parser = argparse.ArgumentParser()
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001356 parser.add_argument(
1357 '--workspace',
1358 type=str,
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001359 default=os.path.abspath(os.path.dirname(__file__)),
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001360 help='The absolute path to your active Bazel workspace.')
Shanqing Cai71445712018-03-12 19:33:52 -07001361 args = parser.parse_args()
1362
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001363 _TF_WORKSPACE_ROOT = args.workspace
1364 _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME)
1365
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001366 # Make a copy of os.environ to be clear when functions and getting and setting
1367 # environment variables.
1368 environ_cp = dict(os.environ)
1369
Mark Daoust44000ad2019-06-18 09:26:26 -07001370 current_bazel_version = check_bazel_version(_TF_MIN_BAZEL_VERSION,
1371 _TF_MAX_BAZEL_VERSION)
A. Unique TensorFlowered297342019-03-15 11:25:28 -07001372 _TF_CURRENT_BAZEL_VERSION = convert_version_to_int(current_bazel_version)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001373
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001374 reset_tf_configure_bazelrc()
Yun Peng03e63a22018-11-07 11:18:53 +01001375
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001376 cleanup_makefile()
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -07001377 setup_python(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001378
1379 if is_windows():
Yifei Fengb1d8c592017-11-22 13:42:21 -08001380 environ_cp['TF_NEED_OPENCL_SYCL'] = '0'
1381 environ_cp['TF_NEED_COMPUTECPP'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001382 environ_cp['TF_NEED_OPENCL'] = '0'
1383 environ_cp['TF_CUDA_CLANG'] = '0'
Guangda Lai76f69382018-01-25 23:59:19 -08001384 environ_cp['TF_NEED_TENSORRT'] = '0'
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001385 # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on
1386 # Windows.
1387 environ_cp['TF_DOWNLOAD_CLANG'] = '0'
A. Unique TensorFlower6e97fb32018-07-16 14:07:29 -07001388 environ_cp['TF_NEED_MPI'] = '0'
1389 environ_cp['TF_SET_ANDROID_WORKSPACE'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001390
1391 if is_macos():
Guangda Lai76f69382018-01-25 23:59:19 -08001392 environ_cp['TF_NEED_TENSORRT'] = '0'
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001393 else:
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001394 environ_cp['TF_CONFIGURE_IOS'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001395
Sanjoy Dasf3fab7c2019-06-20 11:19:00 -07001396 xla_enabled_by_default = is_linux() or is_macos()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001397 set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
Grzegorz Pawelczakec82efd2018-10-09 15:03:46 +01001398 xla_enabled_by_default, 'xla')
Gunhan Gulsoy3da0dff2018-09-26 11:55:50 -07001399
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001400 set_action_env_var(
1401 environ_cp,
1402 'TF_NEED_OPENCL_SYCL',
1403 'OpenCL SYCL',
1404 False,
1405 bazel_config_name='sycl')
Yifei Fengb1d8c592017-11-22 13:42:21 -08001406 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1':
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001407 set_host_cxx_compiler(environ_cp)
1408 set_host_c_compiler(environ_cp)
Yifei Fengb1d8c592017-11-22 13:42:21 -08001409 set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True)
1410 if environ_cp.get('TF_NEED_COMPUTECPP') == '1':
1411 set_computecpp_toolkit_path(environ_cp)
1412 else:
1413 set_trisycl_include_dir(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001414
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001415 set_action_env_var(
1416 environ_cp, 'TF_NEED_ROCM', 'ROCm', False, bazel_config_name='rocm')
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001417 if (environ_cp.get('TF_NEED_ROCM') == '1' and
TensorFlower Gardener62e60162018-09-27 10:22:55 -07001418 'LD_LIBRARY_PATH' in environ_cp and
1419 environ_cp.get('LD_LIBRARY_PATH') != '1'):
1420 write_action_env_to_bazelrc('LD_LIBRARY_PATH',
1421 environ_cp.get('LD_LIBRARY_PATH'))
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001422
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001423 environ_cp['TF_NEED_CUDA'] = str(
1424 int(get_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False)))
A. Unique TensorFlower24cbb2a2017-09-08 07:45:44 -07001425 if (environ_cp.get('TF_NEED_CUDA') == '1' and
1426 'TF_CUDA_CONFIG_REPO' not in environ_cp):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001427
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001428 set_action_env_var(
1429 environ_cp,
1430 'TF_NEED_TENSORRT',
1431 'TensorRT',
1432 False,
1433 bazel_config_name='tensorrt')
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001434
1435 environ_save = dict(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001436 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001437
1438 if validate_cuda_config(environ_cp):
1439 cuda_env_names = [
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001440 'TF_CUDA_VERSION',
1441 'TF_CUBLAS_VERSION',
1442 'TF_CUDNN_VERSION',
1443 'TF_TENSORRT_VERSION',
1444 'TF_NCCL_VERSION',
1445 'TF_CUDA_PATHS',
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001446 # Items below are for backwards compatibility when not using
1447 # TF_CUDA_PATHS.
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001448 'CUDA_TOOLKIT_PATH',
1449 'CUDNN_INSTALL_PATH',
1450 'NCCL_INSTALL_PATH',
1451 'NCCL_HDR_PATH',
1452 'TENSORRT_INSTALL_PATH'
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001453 ]
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001454 # Note: set_action_env_var above already writes to bazelrc.
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001455 for name in cuda_env_names:
1456 if name in environ_cp:
1457 write_action_env_to_bazelrc(name, environ_cp[name])
1458 break
1459
1460 # Restore settings changed below if CUDA config could not be validated.
1461 environ_cp = dict(environ_save)
1462
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001463 set_tf_cuda_version(environ_cp)
1464 set_tf_cudnn_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001465 if is_linux():
1466 set_tf_tensorrt_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001467 set_tf_nccl_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001468
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001469 set_tf_cuda_paths(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001470
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001471 else:
1472 raise UserInputError(
1473 'Invalid CUDA setting were provided %d '
1474 'times in a row. Assuming to be a scripting mistake.' %
1475 _DEFAULT_PROMPT_ASK_ATTEMPTS)
1476
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001477 set_tf_cuda_compute_capabilities(environ_cp)
Ankur Taly0e6f39d2018-02-16 18:22:55 -08001478 if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get(
1479 'LD_LIBRARY_PATH') != '1':
1480 write_action_env_to_bazelrc('LD_LIBRARY_PATH',
1481 environ_cp.get('LD_LIBRARY_PATH'))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001482
1483 set_tf_cuda_clang(environ_cp)
1484 if environ_cp.get('TF_CUDA_CLANG') == '1':
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001485 # Ask whether we should download the clang toolchain.
1486 set_tf_download_clang(environ_cp)
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -08001487 if environ_cp.get('TF_DOWNLOAD_CLANG') != '1':
1488 # Set up which clang we should use as the cuda / host compiler.
1489 set_clang_cuda_compiler_path(environ_cp)
Ilya Biryukov1c3d02e2018-09-04 03:09:52 -07001490 else:
1491 # Use downloaded LLD for linking.
1492 write_to_bazelrc('build:cuda_clang --config=download_clang_use_lld')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001493 else:
1494 # Set up which gcc nvcc should use as the host compiler
1495 # No need to set this on Windows
1496 if not is_windows():
1497 set_gcc_host_compiler_path(environ_cp)
1498 set_other_cuda_vars(environ_cp)
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001499 else:
1500 # CUDA not required. Ask whether we should download the clang toolchain and
1501 # use it for the CPU build.
1502 set_tf_download_clang(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001503
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001504 # SYCL / ROCm / CUDA are mutually exclusive.
1505 # At most 1 GPU platform can be configured.
1506 gpu_platform_count = 0
1507 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1':
1508 gpu_platform_count += 1
1509 if environ_cp.get('TF_NEED_ROCM') == '1':
1510 gpu_platform_count += 1
1511 if environ_cp.get('TF_NEED_CUDA') == '1':
1512 gpu_platform_count += 1
1513 if gpu_platform_count >= 2:
1514 raise UserInputError('SYCL / CUDA / ROCm are mututally exclusive. '
1515 'At most 1 GPU platform can be configured.')
1516
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001517 set_cc_opt_flags(environ_cp)
Yifei Feng5198cb82018-08-17 13:53:06 -07001518 set_system_libs_flag(environ_cp)
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001519 if is_windows():
1520 set_windows_build_flags(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001521
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001522 if get_var(environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', False,
1523 ('Would you like to interactively configure ./WORKSPACE for '
1524 'Android builds?'), 'Searching for NDK and SDK installations.',
1525 'Not configuring the WORKSPACE for Android builds.'):
Michael Case51053502018-06-05 17:47:19 -07001526 create_android_ndk_rule(environ_cp)
1527 create_android_sdk_rule(environ_cp)
Austin Anderson6afface2017-12-05 11:59:17 -08001528
A. Unique TensorFlowere9797fd2019-03-22 11:02:40 -07001529 system_specific_test_config(os.environ)
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001530
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -07001531 set_action_env_var(environ_cp, 'TF_CONFIGURE_IOS', 'iOS', False)
1532 if environ_cp.get('TF_CONFIGURE_IOS') == '1':
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001533 configure_ios()
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001534
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001535 print('Preconfigured Bazel build configs. You can use any of the below by '
1536 'adding "--config=<>" to your build command. See .bazelrc for more '
1537 'details.')
1538 config_info_line('mkl', 'Build with MKL support.')
1539 config_info_line('monolithic', 'Config for mostly static monolithic build.')
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001540 config_info_line('ngraph', 'Build with Intel nGraph support.')
A. Unique TensorFlowera6bf9c82019-02-26 10:08:35 -08001541 config_info_line('numa', 'Build with NUMA support.')
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -08001542 config_info_line(
1543 'dynamic_kernels',
1544 '(Experimental) Build kernels into separate shared objects.')
Anna Raedf6742019-06-12 11:30:51 -07001545 config_info_line('v2', 'Build TensorFlow 2.x instead of 1.x.')
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001546
1547 print('Preconfigured Bazel build configs to DISABLE default on features:')
1548 config_info_line('noaws', 'Disable AWS S3 filesystem support.')
1549 config_info_line('nogcp', 'Disable GCP support.')
1550 config_info_line('nohdfs', 'Disable HDFS support.')
Gunhan Gulsoyeea81682018-11-26 16:51:23 -08001551 config_info_line('nonccl', 'Disable NVIDIA NCCL support.')
Austin Anderson6afface2017-12-05 11:59:17 -08001552
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001553
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001554if __name__ == '__main__':
1555 main()