blob: a55acb10bcbfdcb577ef8dd4c31306aee56c63a7 [file] [log] [blame]
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""configure script to get build parameters from user."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
Shanqing Cai71445712018-03-12 19:33:52 -070021import argparse
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070022import errno
23import os
24import platform
25import re
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070026import subprocess
27import sys
28
Andrew Sellec9885ea2017-11-06 09:37:03 -080029# pylint: disable=g-import-not-at-top
Jonathan Hseu008910f2017-08-25 14:01:05 -070030try:
31 from shutil import which
32except ImportError:
33 from distutils.spawn import find_executable as which
Andrew Sellec9885ea2017-11-06 09:37:03 -080034# pylint: enable=g-import-not-at-top
Jonathan Hseu008910f2017-08-25 14:01:05 -070035
Christian Sigg8dff6b92019-12-09 04:05:46 -080036_DEFAULT_CUDA_VERSION = '10'
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -070037_DEFAULT_CUDNN_VERSION = '7'
Guangda Laifaa93ac2019-10-23 15:07:12 -070038_DEFAULT_TENSORRT_VERSION = '6'
Smit Hinsufe7d1d92018-07-14 13:16:58 -070039_DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0'
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -070040
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070041_TF_OPENCL_VERSION = '1.2'
42_DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp'
Yifei Fengb1d8c592017-11-22 13:42:21 -080043_DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include'
A. Unique TensorFlower82820ef2018-11-12 13:22:13 -080044_SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15, 16, 17, 18]
Austin Anderson6afface2017-12-05 11:59:17 -080045
46_DEFAULT_PROMPT_ASK_ATTEMPTS = 10
47
Shanqing Cai71445712018-03-12 19:33:52 -070048_TF_BAZELRC_FILENAME = '.tf_configure.bazelrc'
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -070049_TF_WORKSPACE_ROOT = ''
50_TF_BAZELRC = ''
A. Unique TensorFlowered297342019-03-15 11:25:28 -070051_TF_CURRENT_BAZEL_VERSION = None
Gunhan Gulsoyaaea5412019-11-29 22:08:00 -080052_TF_MIN_BAZEL_VERSION = '1.0.0'
A. Unique TensorFlowera5f9bcd2019-11-12 07:08:54 -080053_TF_MAX_BAZEL_VERSION = '1.1.0'
Shanqing Cai71445712018-03-12 19:33:52 -070054
Jason Furmanek7c234152018-09-26 04:44:12 +000055NCCL_LIB_PATHS = [
TensorFlower Gardenerea5c5292018-10-01 17:28:26 -070056 'lib64/', 'lib/powerpc64le-linux-gnu/', 'lib/x86_64-linux-gnu/', ''
Jason Furmanek7c234152018-09-26 04:44:12 +000057]
Austin Anderson6afface2017-12-05 11:59:17 -080058
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -070059# List of files to configure when building Bazel on Apple platforms.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -080060APPLE_BAZEL_FILES = [
A. Unique TensorFlower6a059c02019-04-05 14:28:27 -070061 'tensorflow/lite/experimental/ios/BUILD',
A. Unique TensorFlower93e70732019-02-14 16:45:32 -080062 'tensorflow/lite/experimental/objc/BUILD',
YoungSeok Yoon7089aaa2019-11-11 21:36:17 -080063 'tensorflow/lite/experimental/swift/BUILD',
64 'tensorflow/lite/tools/benchmark/experimental/ios/BUILD'
A. Unique TensorFlower93e70732019-02-14 16:45:32 -080065]
66
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -070067# List of files to move when building for iOS.
68IOS_FILES = [
69 'tensorflow/lite/experimental/objc/TensorFlowLiteObjC.podspec',
70 'tensorflow/lite/experimental/swift/TensorFlowLiteSwift.podspec',
71]
72
Austin Anderson6afface2017-12-05 11:59:17 -080073
74class UserInputError(Exception):
75 pass
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070076
77
78def is_windows():
79 return platform.system() == 'Windows'
80
81
82def is_linux():
83 return platform.system() == 'Linux'
84
85
86def is_macos():
87 return platform.system() == 'Darwin'
88
89
90def is_ppc64le():
91 return platform.machine() == 'ppc64le'
92
93
Jonathan Hseu008910f2017-08-25 14:01:05 -070094def is_cygwin():
95 return platform.system().startswith('CYGWIN_NT')
96
97
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070098def get_input(question):
99 try:
100 try:
101 answer = raw_input(question)
102 except NameError:
103 answer = input(question) # pylint: disable=bad-builtin
104 except EOFError:
105 answer = ''
106 return answer
107
108
109def symlink_force(target, link_name):
110 """Force symlink, equivalent of 'ln -sf'.
111
112 Args:
113 target: items to link to.
114 link_name: name of the link.
115 """
116 try:
117 os.symlink(target, link_name)
118 except OSError as e:
119 if e.errno == errno.EEXIST:
120 os.remove(link_name)
121 os.symlink(target, link_name)
122 else:
123 raise e
124
125
126def sed_in_place(filename, old, new):
127 """Replace old string with new string in file.
128
129 Args:
130 filename: string for filename.
131 old: string to replace.
132 new: new string to replace to.
133 """
134 with open(filename, 'r') as f:
135 filedata = f.read()
136 newdata = filedata.replace(old, new)
137 with open(filename, 'w') as f:
138 f.write(newdata)
139
140
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700141def write_to_bazelrc(line):
142 with open(_TF_BAZELRC, 'a') as f:
143 f.write(line + '\n')
144
145
146def write_action_env_to_bazelrc(var_name, var):
147 write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var)))
148
149
Wallyss Lima72371042019-10-03 12:55:30 -0300150def run_shell(cmd, allow_non_zero=False, stderr=None):
151 if stderr is None:
152 stderr = sys.stdout
Jonathan Hseu008910f2017-08-25 14:01:05 -0700153 if allow_non_zero:
154 try:
Wallyss Lima72371042019-10-03 12:55:30 -0300155 output = subprocess.check_output(cmd, stderr=stderr)
Jonathan Hseu008910f2017-08-25 14:01:05 -0700156 except subprocess.CalledProcessError as e:
157 output = e.output
158 else:
Wallyss Lima72371042019-10-03 12:55:30 -0300159 output = subprocess.check_output(cmd, stderr=stderr)
Jonathan Hseu008910f2017-08-25 14:01:05 -0700160 return output.decode('UTF-8').strip()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700161
162
163def cygpath(path):
164 """Convert path from posix to windows."""
Martin Wicked57572e2017-09-02 19:21:45 -0700165 return os.path.abspath(path).replace('\\', '/')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700166
167
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700168def get_python_path(environ_cp, python_bin_path):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700169 """Get the python site package paths."""
170 python_paths = []
171 if environ_cp.get('PYTHONPATH'):
172 python_paths = environ_cp.get('PYTHONPATH').split(':')
173 try:
Wallyss Lima484304e2019-10-03 12:58:20 -0300174 stderr = open(os.devnull, 'wb')
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700175 library_paths = run_shell([
176 python_bin_path, '-c',
177 'import site; print("\\n".join(site.getsitepackages()))'
Wallyss Lima484304e2019-10-03 12:58:20 -0300178 ], stderr=stderr).split('\n')
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700179 except subprocess.CalledProcessError:
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700180 library_paths = [
181 run_shell([
182 python_bin_path, '-c',
183 'from distutils.sysconfig import get_python_lib;'
184 'print(get_python_lib())'
185 ])
186 ]
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700187
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700188 all_paths = set(python_paths + library_paths)
189
190 paths = []
191 for path in all_paths:
192 if os.path.isdir(path):
193 paths.append(path)
194 return paths
195
196
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700197def get_python_major_version(python_bin_path):
198 """Get the python major version."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700199 return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])'])
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700200
201
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -0700202def setup_python(environ_cp):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700203 """Setup python related env variables."""
204 # Get PYTHON_BIN_PATH, default is the current running python.
205 default_python_bin_path = sys.executable
206 ask_python_bin_path = ('Please specify the location of python. [Default is '
207 '%s]: ') % default_python_bin_path
208 while True:
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700209 python_bin_path = get_from_env_or_user_or_default(environ_cp,
210 'PYTHON_BIN_PATH',
211 ask_python_bin_path,
212 default_python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700213 # Check if the path is valid
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700214 if os.path.isfile(python_bin_path) and os.access(python_bin_path, os.X_OK):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700215 break
216 elif not os.path.exists(python_bin_path):
217 print('Invalid python path: %s cannot be found.' % python_bin_path)
218 else:
219 print('%s is not executable. Is it the python binary?' % python_bin_path)
220 environ_cp['PYTHON_BIN_PATH'] = ''
221
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700222 # Convert python path to Windows style before checking lib and version
Martin Wicked57572e2017-09-02 19:21:45 -0700223 if is_windows() or is_cygwin():
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700224 python_bin_path = cygpath(python_bin_path)
225
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700226 # Get PYTHON_LIB_PATH
227 python_lib_path = environ_cp.get('PYTHON_LIB_PATH')
228 if not python_lib_path:
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700229 python_lib_paths = get_python_path(environ_cp, python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700230 if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1':
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700231 python_lib_path = python_lib_paths[0]
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700232 else:
Jonathan Hseu008910f2017-08-25 14:01:05 -0700233 print('Found possible Python library paths:\n %s' %
234 '\n '.join(python_lib_paths))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700235 default_python_lib_path = python_lib_paths[0]
236 python_lib_path = get_input(
Jonathan Hseu008910f2017-08-25 14:01:05 -0700237 'Please input the desired Python library path to use. '
238 'Default is [%s]\n' % python_lib_paths[0])
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700239 if not python_lib_path:
240 python_lib_path = default_python_lib_path
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700241 environ_cp['PYTHON_LIB_PATH'] = python_lib_path
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700242
A. Unique TensorFlowercfccbdb2019-06-24 11:01:58 -0700243 python_major_version = get_python_major_version(python_bin_path)
244 if python_major_version == '2':
245 write_to_bazelrc('build --host_force_python=PY2')
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700246
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700247 # Convert python path to Windows style before writing into bazel.rc
Martin Wicked57572e2017-09-02 19:21:45 -0700248 if is_windows() or is_cygwin():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700249 python_lib_path = cygpath(python_lib_path)
250
251 # Set-up env variables used by python_configure.bzl
252 write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path)
253 write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path)
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -0700254 write_to_bazelrc('build --python_path=\"%s"' % python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700255 environ_cp['PYTHON_BIN_PATH'] = python_bin_path
256
William D. Ironsdcc76a52018-11-20 10:35:18 -0600257 # If choosen python_lib_path is from a path specified in the PYTHONPATH
258 # variable, need to tell bazel to include PYTHONPATH
259 if environ_cp.get('PYTHONPATH'):
260 python_paths = environ_cp.get('PYTHONPATH').split(':')
261 if python_lib_path in python_paths:
TensorFlower Gardener968cd182018-11-28 11:33:16 -0800262 write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH'))
William D. Ironsdcc76a52018-11-20 10:35:18 -0600263
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700264 # Write tools/python_bin_path.sh
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700265 with open(
266 os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'),
267 'w') as f:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700268 f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path)
269
270
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -0700271def reset_tf_configure_bazelrc():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700272 """Reset file that contains customized config settings."""
273 open(_TF_BAZELRC, 'w').close()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700274
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800275
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700276def cleanup_makefile():
277 """Delete any leftover BUILD files from the Makefile build.
278
279 These files could interfere with Bazel parsing.
280 """
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700281 makefile_download_dir = os.path.join(_TF_WORKSPACE_ROOT, 'tensorflow',
282 'contrib', 'makefile', 'downloads')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700283 if os.path.isdir(makefile_download_dir):
284 for root, _, filenames in os.walk(makefile_download_dir):
285 for f in filenames:
286 if f.endswith('BUILD'):
287 os.remove(os.path.join(root, f))
288
289
290def get_var(environ_cp,
291 var_name,
292 query_item,
293 enabled_by_default,
294 question=None,
295 yes_reply=None,
296 no_reply=None):
297 """Get boolean input from user.
298
299 If var_name is not set in env, ask user to enable query_item or not. If the
300 response is empty, use the default.
301
302 Args:
303 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530304 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
305 query_item: string for feature related to the variable, e.g. "CUDA for
306 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700307 enabled_by_default: boolean for default behavior.
308 question: optional string for how to ask for user input.
Michael Cased90054e2018-02-07 14:36:00 -0800309 yes_reply: optional string for reply when feature is enabled.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700310 no_reply: optional string for reply when feature is disabled.
311
312 Returns:
313 boolean value of the variable.
Frank Chenc4ef9272018-01-10 11:36:52 -0800314
315 Raises:
316 UserInputError: if an environment variable is set, but it cannot be
317 interpreted as a boolean indicator, assume that the user has made a
318 scripting error, and will continue to provide invalid input.
319 Raise the error to avoid infinitely looping.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700320 """
321 if not question:
322 question = 'Do you wish to build TensorFlow with %s support?' % query_item
323 if not yes_reply:
324 yes_reply = '%s support will be enabled for TensorFlow.' % query_item
325 if not no_reply:
326 no_reply = 'No %s' % yes_reply
327
328 yes_reply += '\n'
329 no_reply += '\n'
330
331 if enabled_by_default:
332 question += ' [Y/n]: '
333 else:
334 question += ' [y/N]: '
335
336 var = environ_cp.get(var_name)
Frank Chenc4ef9272018-01-10 11:36:52 -0800337 if var is not None:
338 var_content = var.strip().lower()
339 true_strings = ('1', 't', 'true', 'y', 'yes')
340 false_strings = ('0', 'f', 'false', 'n', 'no')
341 if var_content in true_strings:
342 var = True
343 elif var_content in false_strings:
344 var = False
345 else:
346 raise UserInputError(
347 'Environment variable %s must be set as a boolean indicator.\n'
348 'The following are accepted as TRUE : %s.\n'
349 'The following are accepted as FALSE: %s.\n'
A. Unique TensorFlowered297342019-03-15 11:25:28 -0700350 'Current value is %s.' %
351 (var_name, ', '.join(true_strings), ', '.join(false_strings), var))
Frank Chenc4ef9272018-01-10 11:36:52 -0800352
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700353 while var is None:
354 user_input_origin = get_input(question)
355 user_input = user_input_origin.strip().lower()
356 if user_input == 'y':
357 print(yes_reply)
358 var = True
359 elif user_input == 'n':
360 print(no_reply)
361 var = False
362 elif not user_input:
363 if enabled_by_default:
364 print(yes_reply)
365 var = True
366 else:
367 print(no_reply)
368 var = False
369 else:
370 print('Invalid selection: %s' % user_input_origin)
371 return var
372
373
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700374def set_build_var(environ_cp,
375 var_name,
376 query_item,
377 option_name,
378 enabled_by_default,
379 bazel_config_name=None):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700380 """Set if query_item will be enabled for the build.
381
382 Ask user if query_item will be enabled. Default is used if no input is given.
383 Set subprocess environment variable and write to .bazelrc if enabled.
384
385 Args:
386 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530387 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
388 query_item: string for feature related to the variable, e.g. "CUDA for
389 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700390 option_name: string for option to define in .bazelrc.
391 enabled_by_default: boolean for default behavior.
Michael Case98850a52017-09-14 13:35:57 -0700392 bazel_config_name: Name for Bazel --config argument to enable build feature.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700393 """
394
395 var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default)))
396 environ_cp[var_name] = var
397 if var == '1':
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700398 write_to_bazelrc('build:%s --define %s=true' %
399 (bazel_config_name, option_name))
Yifei Fengec451f52018-10-05 12:53:50 -0700400 write_to_bazelrc('build --config=%s' % bazel_config_name)
Michael Case98850a52017-09-14 13:35:57 -0700401 elif bazel_config_name is not None:
402 # TODO(mikecase): Migrate all users of configure.py to use --config Bazel
403 # options and not to set build configs through environment variables.
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700404 write_to_bazelrc('build:%s --define %s=true' %
405 (bazel_config_name, option_name))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700406
407
408def set_action_env_var(environ_cp,
409 var_name,
410 query_item,
411 enabled_by_default,
412 question=None,
413 yes_reply=None,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700414 no_reply=None,
415 bazel_config_name=None):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700416 """Set boolean action_env variable.
417
418 Ask user if query_item will be enabled. Default is used if no input is given.
419 Set environment variable and write to .bazelrc.
420
421 Args:
422 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530423 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
424 query_item: string for feature related to the variable, e.g. "CUDA for
425 Nvidia GPUs".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700426 enabled_by_default: boolean for default behavior.
427 question: optional string for how to ask for user input.
Michael Cased90054e2018-02-07 14:36:00 -0800428 yes_reply: optional string for reply when feature is enabled.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700429 no_reply: optional string for reply when feature is disabled.
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700430 bazel_config_name: adding config to .bazelrc instead of action_env.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700431 """
432 var = int(
433 get_var(environ_cp, var_name, query_item, enabled_by_default, question,
434 yes_reply, no_reply))
435
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700436 if not bazel_config_name:
437 write_action_env_to_bazelrc(var_name, var)
438 elif var:
439 write_to_bazelrc('build --config=%s' % bazel_config_name)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700440 environ_cp[var_name] = str(var)
441
442
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700443def convert_version_to_int(version):
444 """Convert a version number to a integer that can be used to compare.
445
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700446 Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The
447 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored.
448
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700449 Args:
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700450 version: a version to be converted
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700451
452 Returns:
453 An integer if converted successfully, otherwise return None.
454 """
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700455 version = version.split('-')[0]
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700456 version_segments = version.split('.')
Austin Anderson87ea41d2019-04-04 10:03:50 -0700457 # Treat "0.24" as "0.24.0"
458 if len(version_segments) == 2:
459 version_segments.append('0')
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700460 for seg in version_segments:
461 if not seg.isdigit():
462 return None
463
464 version_str = ''.join(['%03d' % int(seg) for seg in version_segments])
465 return int(version_str)
466
467
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800468def check_bazel_version(min_version, max_version):
469 """Check installed bazel version is between min_version and max_version.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700470
471 Args:
Mihai Maruseac4db860f2019-04-19 08:52:10 -0700472 min_version: string for minimum bazel version (must exist!).
473 max_version: string for maximum bazel version (must exist!).
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700474
475 Returns:
476 The bazel version detected.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700477 """
Jonathan Hseu008910f2017-08-25 14:01:05 -0700478 if which('bazel') is None:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700479 print('Cannot find bazel. Please install bazel.')
480 sys.exit(0)
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700481 curr_version = run_shell(
482 ['bazel', '--batch', '--bazelrc=/dev/null', 'version'])
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700483
484 for line in curr_version.split('\n'):
485 if 'Build label: ' in line:
486 curr_version = line.split('Build label: ')[1]
487 break
488
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700489 min_version_int = convert_version_to_int(min_version)
490 curr_version_int = convert_version_to_int(curr_version)
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800491 max_version_int = convert_version_to_int(max_version)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700492
493 # Check if current bazel version can be detected properly.
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700494 if not curr_version_int:
495 print('WARNING: current bazel installation is not a release version.')
496 print('Make sure you are running at least bazel %s' % min_version)
497 return curr_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700498
Michael Cased94271a2017-08-22 17:26:52 -0700499 print('You have bazel %s installed.' % curr_version)
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700500
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700501 if curr_version_int < min_version_int:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700502 print('Please upgrade your bazel installation to version %s or higher to '
503 'build TensorFlow!' % min_version)
Jason Zamanb41761c2018-10-14 11:28:53 +0800504 sys.exit(1)
TensorFlower Gardener78c246b2018-12-13 12:37:42 -0800505 if (curr_version_int > max_version_int and
506 'TF_IGNORE_MAX_BAZEL_VERSION' not in os.environ):
Mihai Maruseace7a123f2018-11-30 13:33:25 -0800507 print('Please downgrade your bazel installation to version %s or lower to '
Mihai Maruseace0963c42018-12-20 14:27:40 -0800508 'build TensorFlow! To downgrade: download the installer for the old '
509 'version (from https://github.com/bazelbuild/bazel/releases) then '
510 'run the installer.' % max_version)
Jason Zamanb41761c2018-10-14 11:28:53 +0800511 sys.exit(1)
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700512 return curr_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700513
514
515def set_cc_opt_flags(environ_cp):
516 """Set up architecture-dependent optimization flags.
517
518 Also append CC optimization flags to bazel.rc..
519
520 Args:
521 environ_cp: copy of the os.environ.
522 """
523 if is_ppc64le():
524 # gcc on ppc64le does not support -march, use mcpu instead
525 default_cc_opt_flags = '-mcpu=native'
A. Unique TensorFlower1bba94a2018-04-04 15:45:20 -0700526 elif is_windows():
527 default_cc_opt_flags = '/arch:AVX'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700528 else:
Justin Lebar9ef04f52018-10-10 18:52:45 -0700529 default_cc_opt_flags = '-march=native -Wno-sign-compare'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700530 question = ('Please specify optimization flags to use during compilation when'
531 ' bazel option "--config=opt" is specified [Default is %s]: '
532 ) % default_cc_opt_flags
533 cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS',
534 question, default_cc_opt_flags)
535 for opt in cc_opt_flags.split():
Michael Case00177422017-11-10 13:14:03 -0800536 write_to_bazelrc('build:opt --copt=%s' % opt)
537 # It should be safe on the same build host.
A. Unique TensorFlower1bba94a2018-04-04 15:45:20 -0700538 if not is_ppc64le() and not is_windows():
Shanqing Cai71445712018-03-12 19:33:52 -0700539 write_to_bazelrc('build:opt --host_copt=-march=native')
Michael Casebb3355d2017-11-09 08:46:31 -0800540 write_to_bazelrc('build:opt --define with_default_optimizations=true')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700541
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700542
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700543def set_tf_cuda_clang(environ_cp):
544 """set TF_CUDA_CLANG action_env.
545
546 Args:
547 environ_cp: copy of the os.environ.
548 """
549 question = 'Do you want to use clang as CUDA compiler?'
550 yes_reply = 'Clang will be used as CUDA compiler.'
551 no_reply = 'nvcc will be used as CUDA compiler.'
552 set_action_env_var(
553 environ_cp,
554 'TF_CUDA_CLANG',
555 None,
556 False,
557 question=question,
558 yes_reply=yes_reply,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700559 no_reply=no_reply,
560 bazel_config_name='cuda_clang')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700561
562
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800563def set_tf_download_clang(environ_cp):
564 """Set TF_DOWNLOAD_CLANG action_env."""
Ilya Biryukov9e651e42018-03-22 05:33:42 -0700565 question = 'Do you wish to download a fresh release of clang? (Experimental)'
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800566 yes_reply = 'Clang will be downloaded and used to compile tensorflow.'
567 no_reply = 'Clang will not be downloaded.'
568 set_action_env_var(
569 environ_cp,
570 'TF_DOWNLOAD_CLANG',
571 None,
572 False,
573 question=question,
574 yes_reply=yes_reply,
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700575 no_reply=no_reply,
576 bazel_config_name='download_clang')
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800577
578
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700579def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var,
580 var_default):
581 """Get var_name either from env, or user or default.
582
583 If var_name has been set as environment variable, use the preset value, else
584 ask for user input. If no input is provided, the default is used.
585
586 Args:
587 environ_cp: copy of the os.environ.
R S Nikhil Krishna05e348b2019-04-18 15:58:12 +0530588 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700589 ask_for_var: string for how to ask for user input.
590 var_default: default value string.
591
592 Returns:
593 string value for var_name
594 """
595 var = environ_cp.get(var_name)
596 if not var:
597 var = get_input(ask_for_var)
Michael Cased94271a2017-08-22 17:26:52 -0700598 print('\n')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700599 if not var:
600 var = var_default
601 return var
602
603
604def set_clang_cuda_compiler_path(environ_cp):
605 """Set CLANG_CUDA_COMPILER_PATH."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700606 default_clang_path = which('clang') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700607 ask_clang_path = ('Please specify which clang should be used as device and '
608 'host compiler. [Default is %s]: ') % default_clang_path
609
610 while True:
611 clang_cuda_compiler_path = get_from_env_or_user_or_default(
612 environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path,
613 default_clang_path)
614 if os.path.exists(clang_cuda_compiler_path):
615 break
616
617 # Reset and retry
618 print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path)
619 environ_cp['CLANG_CUDA_COMPILER_PATH'] = ''
620
621 # Set CLANG_CUDA_COMPILER_PATH
622 environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path
623 write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH',
624 clang_cuda_compiler_path)
625
626
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700627def prompt_loop_or_load_from_env(environ_cp,
628 var_name,
629 var_default,
630 ask_for_var,
631 check_success,
632 error_msg,
633 suppress_default_error=False,
Austin Anderson58b236b2019-10-22 10:36:10 -0700634 resolve_symlinks=False,
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700635 n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS):
Austin Anderson6afface2017-12-05 11:59:17 -0800636 """Loop over user prompts for an ENV param until receiving a valid response.
637
638 For the env param var_name, read from the environment or verify user input
639 until receiving valid input. When done, set var_name in the environ_cp to its
640 new value.
641
642 Args:
643 environ_cp: (Dict) copy of the os.environ.
644 var_name: (String) string for name of environment variable, e.g. "TF_MYVAR".
645 var_default: (String) default value string.
646 ask_for_var: (String) string for how to ask for user input.
647 check_success: (Function) function that takes one argument and returns a
648 boolean. Should return True if the value provided is considered valid. May
649 contain a complex error message if error_msg does not provide enough
650 information. In that case, set suppress_default_error to True.
651 error_msg: (String) String with one and only one '%s'. Formatted with each
652 invalid response upon check_success(input) failure.
653 suppress_default_error: (Bool) Suppress the above error message in favor of
654 one from the check_success function.
Austin Anderson58b236b2019-10-22 10:36:10 -0700655 resolve_symlinks: (Bool) Translate symbolic links into the real filepath.
Austin Anderson6afface2017-12-05 11:59:17 -0800656 n_ask_attempts: (Integer) Number of times to query for valid input before
657 raising an error and quitting.
658
659 Returns:
660 [String] The value of var_name after querying for input.
661
662 Raises:
663 UserInputError: if a query has been attempted n_ask_attempts times without
Frank Chenc4ef9272018-01-10 11:36:52 -0800664 success, assume that the user has made a scripting error, and will
665 continue to provide invalid input. Raise the error to avoid infinitely
666 looping.
Austin Anderson6afface2017-12-05 11:59:17 -0800667 """
668 default = environ_cp.get(var_name) or var_default
669 full_query = '%s [Default is %s]: ' % (
670 ask_for_var,
671 default,
672 )
673
674 for _ in range(n_ask_attempts):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700675 val = get_from_env_or_user_or_default(environ_cp, var_name, full_query,
Austin Anderson6afface2017-12-05 11:59:17 -0800676 default)
677 if check_success(val):
678 break
679 if not suppress_default_error:
680 print(error_msg % val)
681 environ_cp[var_name] = ''
682 else:
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700683 raise UserInputError('Invalid %s setting was provided %d times in a row. '
684 'Assuming to be a scripting mistake.' %
685 (var_name, n_ask_attempts))
Austin Anderson6afface2017-12-05 11:59:17 -0800686
Austin Anderson58b236b2019-10-22 10:36:10 -0700687 if resolve_symlinks and os.path.islink(val):
688 val = os.path.realpath(val)
Austin Anderson6afface2017-12-05 11:59:17 -0800689 environ_cp[var_name] = val
690 return val
691
692
693def create_android_ndk_rule(environ_cp):
694 """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule."""
695 if is_windows() or is_cygwin():
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -0700696 default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' %
697 environ_cp['APPDATA'])
Austin Anderson6afface2017-12-05 11:59:17 -0800698 elif is_macos():
699 default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME']
700 else:
701 default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME']
702
703 def valid_ndk_path(path):
704 return (os.path.exists(path) and
705 os.path.exists(os.path.join(path, 'source.properties')))
706
707 android_ndk_home_path = prompt_loop_or_load_from_env(
708 environ_cp,
709 var_name='ANDROID_NDK_HOME',
710 var_default=default_ndk_path,
711 ask_for_var='Please specify the home path of the Android NDK to use.',
712 check_success=valid_ndk_path,
713 error_msg=('The path %s or its child file "source.properties" '
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700714 'does not exist.'))
Michael Case51053502018-06-05 17:47:19 -0700715 write_action_env_to_bazelrc('ANDROID_NDK_HOME', android_ndk_home_path)
Jared Dukea0104b72019-04-04 12:23:58 -0700716 write_action_env_to_bazelrc(
717 'ANDROID_NDK_API_LEVEL',
718 get_ndk_api_level(environ_cp, android_ndk_home_path))
Austin Anderson6afface2017-12-05 11:59:17 -0800719
720
721def create_android_sdk_rule(environ_cp):
722 """Set Android variables and write Android SDK WORKSPACE rule."""
723 if is_windows() or is_cygwin():
724 default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA'])
725 elif is_macos():
Shashi Shekharc0ff0cc2018-07-17 09:00:24 -0700726 default_sdk_path = '%s/library/Android/Sdk' % environ_cp['HOME']
Austin Anderson6afface2017-12-05 11:59:17 -0800727 else:
728 default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME']
729
730 def valid_sdk_path(path):
731 return (os.path.exists(path) and
732 os.path.exists(os.path.join(path, 'platforms')) and
733 os.path.exists(os.path.join(path, 'build-tools')))
734
735 android_sdk_home_path = prompt_loop_or_load_from_env(
736 environ_cp,
737 var_name='ANDROID_SDK_HOME',
738 var_default=default_sdk_path,
739 ask_for_var='Please specify the home path of the Android SDK to use.',
740 check_success=valid_sdk_path,
741 error_msg=('Either %s does not exist, or it does not contain the '
742 'subdirectories "platforms" and "build-tools".'))
743
744 platforms = os.path.join(android_sdk_home_path, 'platforms')
745 api_levels = sorted(os.listdir(platforms))
746 api_levels = [x.replace('android-', '') for x in api_levels]
747
748 def valid_api_level(api_level):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700749 return os.path.exists(
750 os.path.join(android_sdk_home_path, 'platforms',
751 'android-' + api_level))
Austin Anderson6afface2017-12-05 11:59:17 -0800752
753 android_api_level = prompt_loop_or_load_from_env(
754 environ_cp,
755 var_name='ANDROID_API_LEVEL',
756 var_default=api_levels[-1],
757 ask_for_var=('Please specify the Android SDK API level to use. '
758 '[Available levels: %s]') % api_levels,
759 check_success=valid_api_level,
760 error_msg='Android-%s is not present in the SDK path.')
761
762 build_tools = os.path.join(android_sdk_home_path, 'build-tools')
763 versions = sorted(os.listdir(build_tools))
764
765 def valid_build_tools(version):
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700766 return os.path.exists(
767 os.path.join(android_sdk_home_path, 'build-tools', version))
Austin Anderson6afface2017-12-05 11:59:17 -0800768
769 android_build_tools_version = prompt_loop_or_load_from_env(
770 environ_cp,
771 var_name='ANDROID_BUILD_TOOLS_VERSION',
772 var_default=versions[-1],
773 ask_for_var=('Please specify an Android build tools version to use. '
774 '[Available versions: %s]') % versions,
775 check_success=valid_build_tools,
776 error_msg=('The selected SDK does not have build-tools version %s '
777 'available.'))
778
Michael Case51053502018-06-05 17:47:19 -0700779 write_action_env_to_bazelrc('ANDROID_BUILD_TOOLS_VERSION',
780 android_build_tools_version)
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700781 write_action_env_to_bazelrc('ANDROID_SDK_API_LEVEL', android_api_level)
782 write_action_env_to_bazelrc('ANDROID_SDK_HOME', android_sdk_home_path)
Austin Anderson6afface2017-12-05 11:59:17 -0800783
784
Jared Dukea0104b72019-04-04 12:23:58 -0700785def get_ndk_api_level(environ_cp, android_ndk_home_path):
786 """Gets the appropriate NDK API level to use for the provided Android NDK path."""
787
788 # First check to see if we're using a blessed version of the NDK.
Austin Anderson6afface2017-12-05 11:59:17 -0800789 properties_path = '%s/source.properties' % android_ndk_home_path
790 if is_windows() or is_cygwin():
791 properties_path = cygpath(properties_path)
792 with open(properties_path, 'r') as f:
793 filedata = f.read()
794
795 revision = re.search(r'Pkg.Revision = (\d+)', filedata)
796 if revision:
Jared Dukea0104b72019-04-04 12:23:58 -0700797 ndk_version = revision.group(1)
Michael Case51053502018-06-05 17:47:19 -0700798 else:
799 raise Exception('Unable to parse NDK revision.')
Jared Dukea0104b72019-04-04 12:23:58 -0700800 if int(ndk_version) not in _SUPPORTED_ANDROID_NDK_VERSIONS:
801 print('WARNING: The NDK version in %s is %s, which is not '
802 'supported by Bazel (officially supported versions: %s). Please use '
803 'another version. Compiling Android targets may result in confusing '
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700804 'errors.\n' %
805 (android_ndk_home_path, ndk_version, _SUPPORTED_ANDROID_NDK_VERSIONS))
Jared Dukea0104b72019-04-04 12:23:58 -0700806
807 # Now grab the NDK API level to use. Note that this is different from the
808 # SDK API level, as the NDK API level is effectively the *min* target SDK
809 # version.
810 platforms = os.path.join(android_ndk_home_path, 'platforms')
811 api_levels = sorted(os.listdir(platforms))
812 api_levels = [
813 x.replace('android-', '') for x in api_levels if 'android-' in x
814 ]
815
816 def valid_api_level(api_level):
817 return os.path.exists(
818 os.path.join(android_ndk_home_path, 'platforms',
819 'android-' + api_level))
820
821 android_ndk_api_level = prompt_loop_or_load_from_env(
822 environ_cp,
823 var_name='ANDROID_NDK_API_LEVEL',
Jared Dukea1309f42019-11-15 17:09:35 -0800824 var_default='21', # 21 is required for ARM64 support.
Jared Dukea0104b72019-04-04 12:23:58 -0700825 ask_for_var=('Please specify the (min) Android NDK API level to use. '
826 '[Available levels: %s]') % api_levels,
827 check_success=valid_api_level,
828 error_msg='Android-%s is not present in the NDK path.')
829
830 return android_ndk_api_level
Austin Anderson6afface2017-12-05 11:59:17 -0800831
832
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700833def set_gcc_host_compiler_path(environ_cp):
834 """Set GCC_HOST_COMPILER_PATH."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700835 default_gcc_host_compiler_path = which('gcc') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700836 cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH')
837
838 if os.path.islink(cuda_bin_symlink):
839 # os.readlink is only available in linux
Jonathan Hseu008910f2017-08-25 14:01:05 -0700840 default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700841
Austin Anderson6afface2017-12-05 11:59:17 -0800842 gcc_host_compiler_path = prompt_loop_or_load_from_env(
843 environ_cp,
844 var_name='GCC_HOST_COMPILER_PATH',
845 var_default=default_gcc_host_compiler_path,
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800846 ask_for_var='Please specify which gcc should be used by nvcc as the host compiler.',
Austin Anderson6afface2017-12-05 11:59:17 -0800847 check_success=os.path.exists,
Austin Anderson58b236b2019-10-22 10:36:10 -0700848 resolve_symlinks=True,
Austin Anderson6afface2017-12-05 11:59:17 -0800849 error_msg='Invalid gcc path. %s cannot be found.',
850 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700851
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700852 write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path)
853
854
Ankur Taly0e6f39d2018-02-16 18:22:55 -0800855def reformat_version_sequence(version_str, sequence_count):
856 """Reformat the version string to have the given number of sequences.
857
858 For example:
859 Given (7, 2) -> 7.0
860 (7.0.1, 2) -> 7.0
861 (5, 1) -> 5
862 (5.0.3.2, 1) -> 5
863
864 Args:
865 version_str: String, the version string.
866 sequence_count: int, an integer.
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700867
Ankur Taly0e6f39d2018-02-16 18:22:55 -0800868 Returns:
869 string, reformatted version string.
870 """
871 v = version_str.split('.')
872 if len(v) < sequence_count:
873 v = v + (['0'] * (sequence_count - len(v)))
874
875 return '.'.join(v[:sequence_count])
876
877
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700878def set_tf_cuda_paths(environ_cp):
879 """Set TF_CUDA_PATHS."""
880 ask_cuda_paths = (
881 'Please specify the comma-separated list of base paths to look for CUDA '
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700882 'libraries and headers. [Leave empty to use the default]: ')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700883 tf_cuda_paths = get_from_env_or_user_or_default(environ_cp, 'TF_CUDA_PATHS',
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700884 ask_cuda_paths, '')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700885 if tf_cuda_paths:
886 environ_cp['TF_CUDA_PATHS'] = tf_cuda_paths
887
888
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700889def set_tf_cuda_version(environ_cp):
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700890 """Set TF_CUDA_VERSION."""
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700891 ask_cuda_version = (
892 'Please specify the CUDA SDK version you want to use. '
893 '[Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700894 tf_cuda_version = get_from_env_or_user_or_default(environ_cp,
895 'TF_CUDA_VERSION',
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700896 ask_cuda_version,
897 _DEFAULT_CUDA_VERSION)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700898 environ_cp['TF_CUDA_VERSION'] = tf_cuda_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700899
900
Yifei Fengb1d8c592017-11-22 13:42:21 -0800901def set_tf_cudnn_version(environ_cp):
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700902 """Set TF_CUDNN_VERSION."""
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700903 ask_cudnn_version = (
904 'Please specify the cuDNN version you want to use. '
905 '[Leave empty to default to cuDNN %s]: ') % _DEFAULT_CUDNN_VERSION
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700906 tf_cudnn_version = get_from_env_or_user_or_default(environ_cp,
907 'TF_CUDNN_VERSION',
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700908 ask_cudnn_version,
909 _DEFAULT_CUDNN_VERSION)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700910 environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700911
912
Mingxing Tan1e7b0e42018-06-28 19:13:20 -0700913def is_cuda_compatible(lib, cuda_ver, cudnn_ver):
914 """Check compatibility between given library and cudnn/cudart libraries."""
915 ldd_bin = which('ldd') or '/usr/bin/ldd'
916 ldd_out = run_shell([ldd_bin, lib], True)
917 ldd_out = ldd_out.split(os.linesep)
918 cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$')
919 cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$')
920 cudnn = None
921 cudart = None
922 cudnn_ok = True # assume no cudnn dependency by default
923 cuda_ok = True # assume no cuda dependency by default
924 for line in ldd_out:
925 if 'libcudnn.so' in line:
926 cudnn = cudnn_pattern.search(line)
927 cudnn_ok = False
928 elif 'libcudart.so' in line:
929 cudart = cuda_pattern.search(line)
930 cuda_ok = False
931 if cudnn and len(cudnn.group(1)):
932 cudnn = convert_version_to_int(cudnn.group(1))
933 if cudart and len(cudart.group(1)):
934 cudart = convert_version_to_int(cudart.group(1))
935 if cudnn is not None:
936 cudnn_ok = (cudnn == cudnn_ver)
937 if cudart is not None:
938 cuda_ok = (cudart == cuda_ver)
939 return cudnn_ok and cuda_ok
940
941
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700942def set_tf_tensorrt_version(environ_cp):
943 """Set TF_TENSORRT_VERSION."""
Guangda Lai76f69382018-01-25 23:59:19 -0800944 if not is_linux():
945 raise ValueError('Currently TensorRT is only supported on Linux platform.')
946
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700947 if not int(environ_cp.get('TF_NEED_TENSORRT', False)):
Guangda Lai76f69382018-01-25 23:59:19 -0800948 return
949
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700950 ask_tensorrt_version = (
951 'Please specify the TensorRT version you want to use. '
Guangda Laifaa93ac2019-10-23 15:07:12 -0700952 '[Leave empty to default to TensorRT %s]: ') % _DEFAULT_TENSORRT_VERSION
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -0700953 tf_tensorrt_version = get_from_env_or_user_or_default(
954 environ_cp, 'TF_TENSORRT_VERSION', ask_tensorrt_version,
955 _DEFAULT_TENSORRT_VERSION)
Guangda Lai76f69382018-01-25 23:59:19 -0800956 environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version
Guangda Lai76f69382018-01-25 23:59:19 -0800957
958
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700959def set_tf_nccl_version(environ_cp):
960 """Set TF_NCCL_VERSION."""
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700961 if not is_linux():
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700962 raise ValueError('Currently NCCL is only supported on Linux platform.')
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700963
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700964 if 'TF_NCCL_VERSION' in environ_cp:
965 return
966
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700967 ask_nccl_version = (
A. Unique TensorFlower53faa312018-10-05 08:46:54 -0700968 'Please specify the locally installed NCCL version you want to use. '
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -0700969 '[Leave empty to use http://github.com/nvidia/nccl]: ')
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -0700970 tf_nccl_version = get_from_env_or_user_or_default(environ_cp,
971 'TF_NCCL_VERSION',
972 ask_nccl_version, '')
A. Unique TensorFlower1fda7642018-04-05 03:09:27 -0700973 environ_cp['TF_NCCL_VERSION'] = tf_nccl_version
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -0800974
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -0700975
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700976def get_native_cuda_compute_capabilities(environ_cp):
977 """Get native cuda compute capabilities.
978
979 Args:
980 environ_cp: copy of the os.environ.
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -0700981
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700982 Returns:
983 string of native cuda compute capabilities, separated by comma.
984 """
985 device_query_bin = os.path.join(
986 environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery')
Jonathan Hseu008910f2017-08-25 14:01:05 -0700987 if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK):
988 try:
989 output = run_shell(device_query_bin).split('\n')
990 pattern = re.compile('[0-9]*\\.[0-9]*')
991 output = [pattern.search(x) for x in output if 'Capability' in x]
992 output = ','.join(x.group() for x in output if x is not None)
993 except subprocess.CalledProcessError:
994 output = ''
995 else:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700996 output = ''
997 return output
998
999
1000def set_tf_cuda_compute_capabilities(environ_cp):
1001 """Set TF_CUDA_COMPUTE_CAPABILITIES."""
1002 while True:
1003 native_cuda_compute_capabilities = get_native_cuda_compute_capabilities(
1004 environ_cp)
1005 if not native_cuda_compute_capabilities:
1006 default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES
1007 else:
1008 default_cuda_compute_capabilities = native_cuda_compute_capabilities
1009
1010 ask_cuda_compute_capabilities = (
1011 'Please specify a list of comma-separated '
P Sudeepam52093562019-02-17 17:34:01 +05301012 'CUDA compute capabilities you want to '
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001013 'build with.\nYou can find the compute '
1014 'capability of your device at: '
1015 'https://developer.nvidia.com/cuda-gpus.\nPlease'
1016 ' note that each additional compute '
1017 'capability significantly increases your '
P Sudeepam52093562019-02-17 17:34:01 +05301018 'build time and binary size, and that '
1019 'TensorFlow only supports compute '
P Sudeepam765ceda2019-02-17 17:39:08 +05301020 'capabilities >= 3.5 [Default is: %s]: ' %
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001021 default_cuda_compute_capabilities)
1022 tf_cuda_compute_capabilities = get_from_env_or_user_or_default(
1023 environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES',
1024 ask_cuda_compute_capabilities, default_cuda_compute_capabilities)
1025 # Check whether all capabilities from the input is valid
1026 all_valid = True
Maciejd0f5bc12018-04-30 22:30:58 -05001027 # Remove all whitespace characters before splitting the string
Michael Case51053502018-06-05 17:47:19 -07001028 # that users may insert by accident, as this will result in error
Maciejd0f5bc12018-04-30 22:30:58 -05001029 tf_cuda_compute_capabilities = ''.join(tf_cuda_compute_capabilities.split())
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001030 for compute_capability in tf_cuda_compute_capabilities.split(','):
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001031 m = re.match('[0-9]+.[0-9]+', compute_capability)
1032 if not m:
Austin Anderson32202dc2019-02-19 10:46:27 -08001033 print('Invalid compute capability: %s' % compute_capability)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001034 all_valid = False
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001035 else:
P Sudeepam52093562019-02-17 17:34:01 +05301036 ver = float(m.group(0))
A. Unique TensorFlower8dc2d0e2019-03-12 01:41:05 -07001037 if ver < 3.0:
1038 print('ERROR: TensorFlow only supports CUDA compute capabilities 3.0 '
Austin Anderson32202dc2019-02-19 10:46:27 -08001039 'and higher. Please re-specify the list of compute '
1040 'capabilities excluding version %s.' % ver)
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001041 all_valid = False
A. Unique TensorFlower8dc2d0e2019-03-12 01:41:05 -07001042 if ver < 3.5:
1043 print('WARNING: XLA does not support CUDA compute capabilities '
1044 'lower than 3.5. Disable XLA when running on older GPUs.')
P Sudeepam765ceda2019-02-17 17:39:08 +05301045
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001046 if all_valid:
1047 break
1048
1049 # Reset and Retry
1050 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = ''
1051
1052 # Set TF_CUDA_COMPUTE_CAPABILITIES
1053 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities
1054 write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES',
1055 tf_cuda_compute_capabilities)
1056
1057
1058def set_other_cuda_vars(environ_cp):
1059 """Set other CUDA related variables."""
A. Unique TensorFlowerab391982018-07-11 04:52:49 -07001060 # If CUDA is enabled, always use GPU during build and test.
1061 if environ_cp.get('TF_CUDA_CLANG') == '1':
1062 write_to_bazelrc('build --config=cuda_clang')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001063 else:
A. Unique TensorFlowerab391982018-07-11 04:52:49 -07001064 write_to_bazelrc('build --config=cuda')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001065
1066
1067def set_host_cxx_compiler(environ_cp):
1068 """Set HOST_CXX_COMPILER."""
Jonathan Hseu008910f2017-08-25 14:01:05 -07001069 default_cxx_host_compiler = which('g++') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001070
Austin Anderson6afface2017-12-05 11:59:17 -08001071 host_cxx_compiler = prompt_loop_or_load_from_env(
1072 environ_cp,
1073 var_name='HOST_CXX_COMPILER',
1074 var_default=default_cxx_host_compiler,
1075 ask_for_var=('Please specify which C++ compiler should be used as the '
1076 'host C++ compiler.'),
1077 check_success=os.path.exists,
1078 error_msg='Invalid C++ compiler path. %s cannot be found.',
1079 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001080
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001081 write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler)
1082
1083
1084def set_host_c_compiler(environ_cp):
1085 """Set HOST_C_COMPILER."""
Jonathan Hseu008910f2017-08-25 14:01:05 -07001086 default_c_host_compiler = which('gcc') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001087
Austin Anderson6afface2017-12-05 11:59:17 -08001088 host_c_compiler = prompt_loop_or_load_from_env(
1089 environ_cp,
1090 var_name='HOST_C_COMPILER',
1091 var_default=default_c_host_compiler,
Shanqing Cai71445712018-03-12 19:33:52 -07001092 ask_for_var=('Please specify which C compiler should be used as the host '
Austin Anderson6afface2017-12-05 11:59:17 -08001093 'C compiler.'),
1094 check_success=os.path.exists,
1095 error_msg='Invalid C compiler path. %s cannot be found.',
1096 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001097
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001098 write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler)
1099
1100
1101def set_computecpp_toolkit_path(environ_cp):
1102 """Set COMPUTECPP_TOOLKIT_PATH."""
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001103
Austin Anderson6afface2017-12-05 11:59:17 -08001104 def toolkit_exists(toolkit_path):
1105 """Check if a computecpp toolkit path is valid."""
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001106 if is_linux():
1107 sycl_rt_lib_path = 'lib/libComputeCpp.so'
1108 else:
1109 sycl_rt_lib_path = ''
1110
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001111 sycl_rt_lib_path_full = os.path.join(toolkit_path, sycl_rt_lib_path)
Austin Anderson6afface2017-12-05 11:59:17 -08001112 exists = os.path.exists(sycl_rt_lib_path_full)
1113 if not exists:
1114 print('Invalid SYCL %s library path. %s cannot be found' %
1115 (_TF_OPENCL_VERSION, sycl_rt_lib_path_full))
1116 return exists
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001117
Austin Anderson6afface2017-12-05 11:59:17 -08001118 computecpp_toolkit_path = prompt_loop_or_load_from_env(
1119 environ_cp,
1120 var_name='COMPUTECPP_TOOLKIT_PATH',
1121 var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH,
1122 ask_for_var=(
1123 'Please specify the location where ComputeCpp for SYCL %s is '
1124 'installed.' % _TF_OPENCL_VERSION),
1125 check_success=toolkit_exists,
1126 error_msg='Invalid SYCL compiler path. %s cannot be found.',
1127 suppress_default_error=True)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001128
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001129 write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH',
1130 computecpp_toolkit_path)
1131
Michael Cased31531a2018-01-05 14:09:41 -08001132
Dandelion Man?90e42f32017-12-15 18:15:07 -08001133def set_trisycl_include_dir(environ_cp):
Michael Cased31531a2018-01-05 14:09:41 -08001134 """Set TRISYCL_INCLUDE_DIR."""
Frank Chenc4ef9272018-01-10 11:36:52 -08001135
Dandelion Man?90e42f32017-12-15 18:15:07 -08001136 ask_trisycl_include_dir = ('Please specify the location of the triSYCL '
1137 'include directory. (Use --config=sycl_trisycl '
1138 'when building with Bazel) '
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001139 '[Default is %s]: ') % (
1140 _DEFAULT_TRISYCL_INCLUDE_DIR)
Frank Chenc4ef9272018-01-10 11:36:52 -08001141
Dandelion Man?90e42f32017-12-15 18:15:07 -08001142 while True:
1143 trisycl_include_dir = get_from_env_or_user_or_default(
Michael Cased31531a2018-01-05 14:09:41 -08001144 environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir,
1145 _DEFAULT_TRISYCL_INCLUDE_DIR)
Dandelion Man?90e42f32017-12-15 18:15:07 -08001146 if os.path.exists(trisycl_include_dir):
1147 break
1148
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001149 print('Invalid triSYCL include directory, %s cannot be found' %
1150 (trisycl_include_dir))
Dandelion Man?90e42f32017-12-15 18:15:07 -08001151
1152 # Set TRISYCL_INCLUDE_DIR
1153 environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001154 write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', trisycl_include_dir)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001155
Yifei Fengb1d8c592017-11-22 13:42:21 -08001156
A. Unique TensorFlowere9797fd2019-03-22 11:02:40 -07001157def system_specific_test_config(env):
A. Unique TensorFlower7bd86372019-03-21 15:19:30 -07001158 """Add default build and test flags required for TF tests to bazelrc."""
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001159 write_to_bazelrc('test --flaky_test_attempts=3')
1160 write_to_bazelrc('test --test_size_filters=small,medium')
frreiss8b013a72019-11-20 11:36:27 -08001161
1162 # Each instance of --test_tag_filters or --build_tag_filters overrides all
1163 # previous instances, so we need to build up a complete list and write a
1164 # single list of filters for the .bazelrc file.
1165
1166 # Filters to use with both --test_tag_filters and --build_tag_filters
frreissbab7d7d2019-11-21 10:52:55 -08001167 test_and_build_filters = ['-benchmark-test', '-no_oss']
1168 # Additional filters for --test_tag_filters beyond those in
1169 # test_and_build_filters
1170 test_only_filters = ['-oss_serial']
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001171 if is_windows():
frreissbab7d7d2019-11-21 10:52:55 -08001172 test_and_build_filters.append('-no_windows')
Guangda Laibcd701a2019-03-12 21:04:51 -07001173 if env.get('TF_NEED_CUDA', None) == '1':
frreissbab7d7d2019-11-21 10:52:55 -08001174 test_and_build_filters += ['-no_windows_gpu', '-no_gpu']
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001175 else:
frreissbab7d7d2019-11-21 10:52:55 -08001176 test_and_build_filters.append('-gpu')
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001177 elif is_macos():
frreissbab7d7d2019-11-21 10:52:55 -08001178 test_and_build_filters += ['-gpu', '-nomac', '-no_mac']
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001179 elif is_linux():
Guangda Laibcd701a2019-03-12 21:04:51 -07001180 if env.get('TF_NEED_CUDA', None) == '1':
frreissbab7d7d2019-11-21 10:52:55 -08001181 test_and_build_filters.append('-no_gpu')
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001182 write_to_bazelrc('test --test_env=LD_LIBRARY_PATH')
1183 else:
frreissbab7d7d2019-11-21 10:52:55 -08001184 test_and_build_filters.append('-gpu')
frreiss3adf4172019-12-06 14:16:49 -08001185
1186 # Disable tests with "v1only" tag in "v2" Bazel config, but not in "v1" config
1187 write_to_bazelrc('test:v1 --test_tag_filters=%s' %
TensorFlower Gardenere5159082019-11-22 11:47:38 -08001188 ','.join(test_and_build_filters + test_only_filters))
frreiss3adf4172019-12-06 14:16:49 -08001189 write_to_bazelrc('test:v1 --build_tag_filters=%s' %
TensorFlower Gardenere5159082019-11-22 11:47:38 -08001190 ','.join(test_and_build_filters))
TensorFlower Gardener26a1de32019-12-17 07:11:48 -08001191 write_to_bazelrc(
1192 'test:v2 --test_tag_filters=%s' %
1193 ','.join(test_and_build_filters + test_only_filters + ['-v1only']))
frreiss3adf4172019-12-06 14:16:49 -08001194 write_to_bazelrc('test:v2 --build_tag_filters=%s' %
TensorFlower Gardener26a1de32019-12-17 07:11:48 -08001195 ','.join(test_and_build_filters + ['-v1only']))
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001196
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001197
Yifei Feng5198cb82018-08-17 13:53:06 -07001198def set_system_libs_flag(environ_cp):
1199 syslibs = environ_cp.get('TF_SYSTEM_LIBS', '')
TensorFlower Gardener61a87202018-10-01 12:25:39 -07001200 if syslibs:
Jason Zaman5fc39bd2018-09-16 01:38:55 +08001201 if ',' in syslibs:
1202 syslibs = ','.join(sorted(syslibs.split(',')))
1203 else:
1204 syslibs = ','.join(sorted(syslibs.split()))
Yifei Feng5198cb82018-08-17 13:53:06 -07001205 write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs)
1206
Jason Zaman5fc39bd2018-09-16 01:38:55 +08001207 if 'PREFIX' in environ_cp:
1208 write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX'])
1209 if 'LIBDIR' in environ_cp:
1210 write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR'])
1211 if 'INCLUDEDIR' in environ_cp:
1212 write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR'])
1213
Yifei Feng5198cb82018-08-17 13:53:06 -07001214
Austin Anderson6d7c3762019-11-19 17:04:19 -08001215def is_reduced_optimize_huge_functions_available(environ_cp):
1216 """Check to see if the system supports /d2ReducedOptimizeHugeFunctions.
1217
1218 The above compiler flag is a new compiler flag introduced to the Visual Studio
1219 compiler in version 16.4 (available in Visual Studio 2019, Preview edition
1220 only, as of 2019-11-19). TensorFlow needs this flag to massively reduce
1221 compile times, but until 16.4 is officially released, we can't depend on it.
1222
1223 See also https://groups.google.com/a/tensorflow.org/g/build/c/SsW98Eo7l3o
1224
1225 Because it's very annoying to check this manually (to check the MSVC installed
1226 versions, you need to use the registry, and it's not clear if Bazel will be
1227 using that install version anyway), we expect enviroments who know they may
1228 use this flag to export TF_VC_VERSION=16.4
1229
1230 TODO(angerson, gunan): Remove this function when TensorFlow's minimum VS
1231 version is upgraded to 16.4.
1232
1233 Arguments:
1234 environ_cp: Environment of the current execution
1235
1236 Returns:
1237 boolean, whether or not /d2ReducedOptimizeHugeFunctions is available on this
1238 machine.
1239 """
1240 return float(environ_cp.get('TF_VC_VERSION', '0')) >= 16.4
1241
1242
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001243def set_windows_build_flags(environ_cp):
1244 """Set Windows specific build options."""
Austin Anderson6d7c3762019-11-19 17:04:19 -08001245 if is_reduced_optimize_huge_functions_available(environ_cp):
1246 write_to_bazelrc(
1247 'build --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions'
1248 )
1249
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001250 if get_var(
1251 environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline',
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001252 True, ('Would you like to override eigen strong inline for some C++ '
1253 'compilation to reduce the compilation time?'),
1254 'Eigen strong inline overridden.', 'Not overriding eigen strong inline, '
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001255 'some compilations could take more than 20 mins.'):
1256 # Due to a known MSVC compiler issue
1257 # https://github.com/tensorflow/tensorflow/issues/10521
1258 # Overriding eigen strong inline speeds up the compiling of
1259 # conv_grad_ops_3d.cc and conv_ops_3d.cc by 20 minutes,
1260 # but this also hurts the performance. Let users decide what they want.
1261 write_to_bazelrc('build --define=override_eigen_strong_inline=true')
Dandelion Man?90e42f32017-12-15 18:15:07 -08001262
A. Unique TensorFlower061c3592017-11-13 14:21:04 -08001263
Michael Cased31531a2018-01-05 14:09:41 -08001264def config_info_line(name, help_text):
1265 """Helper function to print formatted help text for Bazel config options."""
1266 print('\t--config=%-12s\t# %s' % (name, help_text))
1267
1268
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001269def configure_ios():
1270 """Configures TensorFlow for iOS builds.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001271
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001272 This function will only be executed if `is_macos()` is true.
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001273 """
1274 if not is_macos():
1275 return
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001276 for filepath in APPLE_BAZEL_FILES:
1277 existing_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath + '.apple')
1278 renamed_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath)
1279 symlink_force(existing_filepath, renamed_filepath)
1280 for filepath in IOS_FILES:
1281 filename = os.path.basename(filepath)
1282 new_filepath = os.path.join(_TF_WORKSPACE_ROOT, filename)
1283 symlink_force(filepath, new_filepath)
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001284
1285
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001286def validate_cuda_config(environ_cp):
1287 """Run find_cuda_config.py and return cuda_toolkit_path, or None."""
1288
1289 def maybe_encode_env(env):
1290 """Encodes unicode in env to str on Windows python 2.x."""
1291 if not is_windows() or sys.version_info[0] != 2:
1292 return env
1293 for k, v in env.items():
1294 if isinstance(k, unicode):
1295 k = k.encode('ascii')
1296 if isinstance(v, unicode):
1297 v = v.encode('ascii')
1298 env[k] = v
1299 return env
1300
1301 cuda_libraries = ['cuda', 'cudnn']
1302 if is_linux():
A. Unique TensorFlower28fc9cc2019-05-01 14:17:54 -07001303 if int(environ_cp.get('TF_NEED_TENSORRT', False)):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001304 cuda_libraries.append('tensorrt')
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001305 if environ_cp.get('TF_NCCL_VERSION', None):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001306 cuda_libraries.append('nccl')
1307
1308 proc = subprocess.Popen(
1309 [environ_cp['PYTHON_BIN_PATH'], 'third_party/gpus/find_cuda_config.py'] +
1310 cuda_libraries,
1311 stdout=subprocess.PIPE,
1312 env=maybe_encode_env(environ_cp))
1313
1314 if proc.wait():
1315 # Errors from find_cuda_config.py were sent to stderr.
A. Unique TensorFlower0cb8a072019-04-16 08:16:39 -07001316 print('Asking for detailed CUDA configuration...\n')
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001317 return False
1318
1319 config = dict(
1320 tuple(line.decode('ascii').rstrip().split(': ')) for line in proc.stdout)
1321
1322 print('Found CUDA %s in:' % config['cuda_version'])
1323 print(' %s' % config['cuda_library_dir'])
1324 print(' %s' % config['cuda_include_dir'])
1325
1326 print('Found cuDNN %s in:' % config['cudnn_version'])
1327 print(' %s' % config['cudnn_library_dir'])
1328 print(' %s' % config['cudnn_include_dir'])
1329
1330 if 'tensorrt_version' in config:
1331 print('Found TensorRT %s in:' % config['tensorrt_version'])
1332 print(' %s' % config['tensorrt_library_dir'])
1333 print(' %s' % config['tensorrt_include_dir'])
1334
1335 if config.get('nccl_version', None):
1336 print('Found NCCL %s in:' % config['nccl_version'])
1337 print(' %s' % config['nccl_library_dir'])
1338 print(' %s' % config['nccl_include_dir'])
1339
1340 print('\n')
1341
1342 environ_cp['CUDA_TOOLKIT_PATH'] = config['cuda_toolkit_path']
1343 return True
1344
1345
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001346def main():
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001347 global _TF_WORKSPACE_ROOT
1348 global _TF_BAZELRC
A. Unique TensorFlowered297342019-03-15 11:25:28 -07001349 global _TF_CURRENT_BAZEL_VERSION
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001350
Shanqing Cai71445712018-03-12 19:33:52 -07001351 parser = argparse.ArgumentParser()
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001352 parser.add_argument(
1353 '--workspace',
1354 type=str,
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001355 default=os.path.abspath(os.path.dirname(__file__)),
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001356 help='The absolute path to your active Bazel workspace.')
Shanqing Cai71445712018-03-12 19:33:52 -07001357 args = parser.parse_args()
1358
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001359 _TF_WORKSPACE_ROOT = args.workspace
1360 _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME)
1361
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001362 # Make a copy of os.environ to be clear when functions and getting and setting
1363 # environment variables.
1364 environ_cp = dict(os.environ)
1365
Mark Daoust44000ad2019-06-18 09:26:26 -07001366 current_bazel_version = check_bazel_version(_TF_MIN_BAZEL_VERSION,
1367 _TF_MAX_BAZEL_VERSION)
A. Unique TensorFlowered297342019-03-15 11:25:28 -07001368 _TF_CURRENT_BAZEL_VERSION = convert_version_to_int(current_bazel_version)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001369
A. Unique TensorFlower7c5eb352018-10-01 07:15:23 -07001370 reset_tf_configure_bazelrc()
Yun Peng03e63a22018-11-07 11:18:53 +01001371
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001372 cleanup_makefile()
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -07001373 setup_python(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001374
1375 if is_windows():
Yifei Fengb1d8c592017-11-22 13:42:21 -08001376 environ_cp['TF_NEED_OPENCL_SYCL'] = '0'
1377 environ_cp['TF_NEED_COMPUTECPP'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001378 environ_cp['TF_NEED_OPENCL'] = '0'
1379 environ_cp['TF_CUDA_CLANG'] = '0'
Guangda Lai76f69382018-01-25 23:59:19 -08001380 environ_cp['TF_NEED_TENSORRT'] = '0'
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001381 # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on
1382 # Windows.
1383 environ_cp['TF_DOWNLOAD_CLANG'] = '0'
A. Unique TensorFlower6e97fb32018-07-16 14:07:29 -07001384 environ_cp['TF_NEED_MPI'] = '0'
1385 environ_cp['TF_SET_ANDROID_WORKSPACE'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001386
1387 if is_macos():
Guangda Lai76f69382018-01-25 23:59:19 -08001388 environ_cp['TF_NEED_TENSORRT'] = '0'
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001389 else:
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001390 environ_cp['TF_CONFIGURE_IOS'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001391
Sanjoy Dasf3fab7c2019-06-20 11:19:00 -07001392 xla_enabled_by_default = is_linux() or is_macos()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001393 set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
Grzegorz Pawelczakec82efd2018-10-09 15:03:46 +01001394 xla_enabled_by_default, 'xla')
Gunhan Gulsoy3da0dff2018-09-26 11:55:50 -07001395
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001396 set_action_env_var(
1397 environ_cp,
1398 'TF_NEED_OPENCL_SYCL',
1399 'OpenCL SYCL',
1400 False,
1401 bazel_config_name='sycl')
Yifei Fengb1d8c592017-11-22 13:42:21 -08001402 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1':
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001403 set_host_cxx_compiler(environ_cp)
1404 set_host_c_compiler(environ_cp)
Yifei Fengb1d8c592017-11-22 13:42:21 -08001405 set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True)
1406 if environ_cp.get('TF_NEED_COMPUTECPP') == '1':
1407 set_computecpp_toolkit_path(environ_cp)
1408 else:
1409 set_trisycl_include_dir(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001410
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001411 set_action_env_var(
1412 environ_cp, 'TF_NEED_ROCM', 'ROCm', False, bazel_config_name='rocm')
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001413 if (environ_cp.get('TF_NEED_ROCM') == '1' and
TensorFlower Gardener62e60162018-09-27 10:22:55 -07001414 'LD_LIBRARY_PATH' in environ_cp and
1415 environ_cp.get('LD_LIBRARY_PATH') != '1'):
1416 write_action_env_to_bazelrc('LD_LIBRARY_PATH',
1417 environ_cp.get('LD_LIBRARY_PATH'))
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001418
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001419 environ_cp['TF_NEED_CUDA'] = str(
1420 int(get_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False)))
A. Unique TensorFlower24cbb2a2017-09-08 07:45:44 -07001421 if (environ_cp.get('TF_NEED_CUDA') == '1' and
1422 'TF_CUDA_CONFIG_REPO' not in environ_cp):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001423
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001424 set_action_env_var(
1425 environ_cp,
1426 'TF_NEED_TENSORRT',
1427 'TensorRT',
1428 False,
1429 bazel_config_name='tensorrt')
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001430
1431 environ_save = dict(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001432 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS):
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001433
1434 if validate_cuda_config(environ_cp):
1435 cuda_env_names = [
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001436 'TF_CUDA_VERSION',
1437 'TF_CUBLAS_VERSION',
1438 'TF_CUDNN_VERSION',
1439 'TF_TENSORRT_VERSION',
1440 'TF_NCCL_VERSION',
1441 'TF_CUDA_PATHS',
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001442 # Items below are for backwards compatibility when not using
1443 # TF_CUDA_PATHS.
A. Unique TensorFlowerfef51672019-05-23 04:19:20 -07001444 'CUDA_TOOLKIT_PATH',
1445 'CUDNN_INSTALL_PATH',
1446 'NCCL_INSTALL_PATH',
1447 'NCCL_HDR_PATH',
1448 'TENSORRT_INSTALL_PATH'
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001449 ]
A. Unique TensorFlowerb2e7f672019-04-30 09:20:36 -07001450 # Note: set_action_env_var above already writes to bazelrc.
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001451 for name in cuda_env_names:
1452 if name in environ_cp:
1453 write_action_env_to_bazelrc(name, environ_cp[name])
1454 break
1455
1456 # Restore settings changed below if CUDA config could not be validated.
1457 environ_cp = dict(environ_save)
1458
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001459 set_tf_cuda_version(environ_cp)
1460 set_tf_cudnn_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001461 if is_linux():
1462 set_tf_tensorrt_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001463 set_tf_nccl_version(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001464
A. Unique TensorFlower651b0bc2019-04-16 06:13:43 -07001465 set_tf_cuda_paths(environ_cp)
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001466
A. Unique TensorFlower8e9ca662019-04-15 12:47:11 -07001467 else:
1468 raise UserInputError(
1469 'Invalid CUDA setting were provided %d '
1470 'times in a row. Assuming to be a scripting mistake.' %
1471 _DEFAULT_PROMPT_ASK_ATTEMPTS)
1472
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001473 set_tf_cuda_compute_capabilities(environ_cp)
Ankur Taly0e6f39d2018-02-16 18:22:55 -08001474 if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get(
1475 'LD_LIBRARY_PATH') != '1':
1476 write_action_env_to_bazelrc('LD_LIBRARY_PATH',
1477 environ_cp.get('LD_LIBRARY_PATH'))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001478
1479 set_tf_cuda_clang(environ_cp)
1480 if environ_cp.get('TF_CUDA_CLANG') == '1':
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001481 # Ask whether we should download the clang toolchain.
1482 set_tf_download_clang(environ_cp)
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -08001483 if environ_cp.get('TF_DOWNLOAD_CLANG') != '1':
1484 # Set up which clang we should use as the cuda / host compiler.
1485 set_clang_cuda_compiler_path(environ_cp)
Ilya Biryukov1c3d02e2018-09-04 03:09:52 -07001486 else:
1487 # Use downloaded LLD for linking.
1488 write_to_bazelrc('build:cuda_clang --config=download_clang_use_lld')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001489 else:
1490 # Set up which gcc nvcc should use as the host compiler
1491 # No need to set this on Windows
1492 if not is_windows():
1493 set_gcc_host_compiler_path(environ_cp)
1494 set_other_cuda_vars(environ_cp)
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001495 else:
1496 # CUDA not required. Ask whether we should download the clang toolchain and
1497 # use it for the CPU build.
1498 set_tf_download_clang(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001499
Wen-Heng (Jack) Chung69d3b8f2018-06-22 23:09:43 -05001500 # SYCL / ROCm / CUDA are mutually exclusive.
1501 # At most 1 GPU platform can be configured.
1502 gpu_platform_count = 0
1503 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1':
1504 gpu_platform_count += 1
1505 if environ_cp.get('TF_NEED_ROCM') == '1':
1506 gpu_platform_count += 1
1507 if environ_cp.get('TF_NEED_CUDA') == '1':
1508 gpu_platform_count += 1
1509 if gpu_platform_count >= 2:
1510 raise UserInputError('SYCL / CUDA / ROCm are mututally exclusive. '
1511 'At most 1 GPU platform can be configured.')
1512
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001513 set_cc_opt_flags(environ_cp)
Yifei Feng5198cb82018-08-17 13:53:06 -07001514 set_system_libs_flag(environ_cp)
A. Unique TensorFlower1b212352018-07-19 13:48:50 -07001515 if is_windows():
1516 set_windows_build_flags(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001517
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001518 if get_var(environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', False,
1519 ('Would you like to interactively configure ./WORKSPACE for '
1520 'Android builds?'), 'Searching for NDK and SDK installations.',
1521 'Not configuring the WORKSPACE for Android builds.'):
Michael Case51053502018-06-05 17:47:19 -07001522 create_android_ndk_rule(environ_cp)
1523 create_android_sdk_rule(environ_cp)
Austin Anderson6afface2017-12-05 11:59:17 -08001524
A. Unique TensorFlowere9797fd2019-03-22 11:02:40 -07001525 system_specific_test_config(os.environ)
Gunhan Gulsoyfff66962019-02-20 15:28:43 -08001526
A. Unique TensorFlower5001ac02019-04-03 11:59:08 -07001527 set_action_env_var(environ_cp, 'TF_CONFIGURE_IOS', 'iOS', False)
1528 if environ_cp.get('TF_CONFIGURE_IOS') == '1':
A. Unique TensorFlower76e879d2019-03-22 00:25:36 -07001529 configure_ios()
A. Unique TensorFlower93e70732019-02-14 16:45:32 -08001530
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001531 print('Preconfigured Bazel build configs. You can use any of the below by '
1532 'adding "--config=<>" to your build command. See .bazelrc for more '
1533 'details.')
1534 config_info_line('mkl', 'Build with MKL support.')
1535 config_info_line('monolithic', 'Config for mostly static monolithic build.')
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001536 config_info_line('ngraph', 'Build with Intel nGraph support.')
A. Unique TensorFlowera6bf9c82019-02-26 10:08:35 -08001537 config_info_line('numa', 'Build with NUMA support.')
TensorFlower Gardenerabb763b2019-02-06 16:23:03 -08001538 config_info_line(
1539 'dynamic_kernels',
1540 '(Experimental) Build kernels into separate shared objects.')
Anna Raedf6742019-06-12 11:30:51 -07001541 config_info_line('v2', 'Build TensorFlow 2.x instead of 1.x.')
Gunhan Gulsoy34370982018-10-12 15:26:01 -07001542
1543 print('Preconfigured Bazel build configs to DISABLE default on features:')
1544 config_info_line('noaws', 'Disable AWS S3 filesystem support.')
1545 config_info_line('nogcp', 'Disable GCP support.')
1546 config_info_line('nohdfs', 'Disable HDFS support.')
Gunhan Gulsoyeea81682018-11-26 16:51:23 -08001547 config_info_line('nonccl', 'Disable NVIDIA NCCL support.')
Austin Anderson6afface2017-12-05 11:59:17 -08001548
Gunhan Gulsoyffa90fc2018-09-26 01:38:55 -07001549
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001550if __name__ == '__main__':
1551 main()