blob: 22b9abedd77abf714c03752eeef0f17da2394e05 [file] [log] [blame]
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""configure script to get build parameters from user."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
Shanqing Cai71445712018-03-12 19:33:52 -070021import argparse
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070022import errno
23import os
24import platform
25import re
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070026import subprocess
27import sys
28
Andrew Sellec9885ea2017-11-06 09:37:03 -080029# pylint: disable=g-import-not-at-top
Jonathan Hseu008910f2017-08-25 14:01:05 -070030try:
31 from shutil import which
32except ImportError:
33 from distutils.spawn import find_executable as which
Andrew Sellec9885ea2017-11-06 09:37:03 -080034# pylint: enable=g-import-not-at-top
Jonathan Hseu008910f2017-08-25 14:01:05 -070035
Dandelion Man?90e42f32017-12-15 18:15:07 -080036_DEFAULT_CUDA_VERSION = '9.0'
37_DEFAULT_CUDNN_VERSION = '7'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070038_DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,5.2'
39_DEFAULT_CUDA_PATH = '/usr/local/cuda'
40_DEFAULT_CUDA_PATH_LINUX = '/opt/cuda'
41_DEFAULT_CUDA_PATH_WIN = ('C:/Program Files/NVIDIA GPU Computing '
42 'Toolkit/CUDA/v%s' % _DEFAULT_CUDA_VERSION)
Benoit Steiner0dadbfe2018-03-22 18:54:27 -070043_DEFAULT_TENSORRT_PATH_LINUX = '/usr/lib/%s-linux-gnu' % platform.machine()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070044_TF_OPENCL_VERSION = '1.2'
45_DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp'
Yifei Fengb1d8c592017-11-22 13:42:21 -080046_DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include'
Austin Anderson6afface2017-12-05 11:59:17 -080047_SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15]
48
49_DEFAULT_PROMPT_ASK_ATTEMPTS = 10
50
Shanqing Cai71445712018-03-12 19:33:52 -070051_TF_WORKSPACE_ROOT = os.path.abspath(os.path.dirname(__file__))
52_TF_BAZELRC_FILENAME = '.tf_configure.bazelrc'
53_TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME)
54_TF_WORKSPACE = os.path.join(_TF_WORKSPACE_ROOT, 'WORKSPACE')
55
Austin Anderson6afface2017-12-05 11:59:17 -080056
57class UserInputError(Exception):
58 pass
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070059
60
61def is_windows():
62 return platform.system() == 'Windows'
63
64
65def is_linux():
66 return platform.system() == 'Linux'
67
68
69def is_macos():
70 return platform.system() == 'Darwin'
71
72
73def is_ppc64le():
74 return platform.machine() == 'ppc64le'
75
76
Jonathan Hseu008910f2017-08-25 14:01:05 -070077def is_cygwin():
78 return platform.system().startswith('CYGWIN_NT')
79
80
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070081def get_input(question):
82 try:
83 try:
84 answer = raw_input(question)
85 except NameError:
86 answer = input(question) # pylint: disable=bad-builtin
87 except EOFError:
88 answer = ''
89 return answer
90
91
92def symlink_force(target, link_name):
93 """Force symlink, equivalent of 'ln -sf'.
94
95 Args:
96 target: items to link to.
97 link_name: name of the link.
98 """
99 try:
100 os.symlink(target, link_name)
101 except OSError as e:
102 if e.errno == errno.EEXIST:
103 os.remove(link_name)
104 os.symlink(target, link_name)
105 else:
106 raise e
107
108
109def sed_in_place(filename, old, new):
110 """Replace old string with new string in file.
111
112 Args:
113 filename: string for filename.
114 old: string to replace.
115 new: new string to replace to.
116 """
117 with open(filename, 'r') as f:
118 filedata = f.read()
119 newdata = filedata.replace(old, new)
120 with open(filename, 'w') as f:
121 f.write(newdata)
122
123
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700124def write_to_bazelrc(line):
125 with open(_TF_BAZELRC, 'a') as f:
126 f.write(line + '\n')
127
128
129def write_action_env_to_bazelrc(var_name, var):
130 write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var)))
131
132
Jonathan Hseu008910f2017-08-25 14:01:05 -0700133def run_shell(cmd, allow_non_zero=False):
134 if allow_non_zero:
135 try:
136 output = subprocess.check_output(cmd)
137 except subprocess.CalledProcessError as e:
138 output = e.output
139 else:
140 output = subprocess.check_output(cmd)
141 return output.decode('UTF-8').strip()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700142
143
144def cygpath(path):
145 """Convert path from posix to windows."""
Martin Wicked57572e2017-09-02 19:21:45 -0700146 return os.path.abspath(path).replace('\\', '/')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700147
148
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700149def get_python_path(environ_cp, python_bin_path):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700150 """Get the python site package paths."""
151 python_paths = []
152 if environ_cp.get('PYTHONPATH'):
153 python_paths = environ_cp.get('PYTHONPATH').split(':')
154 try:
Jonathan Hseu008910f2017-08-25 14:01:05 -0700155 library_paths = run_shell(
156 [python_bin_path, '-c',
Austin Anderson6afface2017-12-05 11:59:17 -0800157 'import site; print("\\n".join(site.getsitepackages()))']).split('\n')
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700158 except subprocess.CalledProcessError:
Jonathan Hseu008910f2017-08-25 14:01:05 -0700159 library_paths = [run_shell(
160 [python_bin_path, '-c',
161 'from distutils.sysconfig import get_python_lib;'
162 'print(get_python_lib())'])]
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700163
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700164 all_paths = set(python_paths + library_paths)
165
166 paths = []
167 for path in all_paths:
168 if os.path.isdir(path):
169 paths.append(path)
170 return paths
171
172
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700173def get_python_major_version(python_bin_path):
174 """Get the python major version."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700175 return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])'])
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700176
177
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -0700178def setup_python(environ_cp):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700179 """Setup python related env variables."""
180 # Get PYTHON_BIN_PATH, default is the current running python.
181 default_python_bin_path = sys.executable
182 ask_python_bin_path = ('Please specify the location of python. [Default is '
183 '%s]: ') % default_python_bin_path
184 while True:
185 python_bin_path = get_from_env_or_user_or_default(
186 environ_cp, 'PYTHON_BIN_PATH', ask_python_bin_path,
187 default_python_bin_path)
188 # Check if the path is valid
Jonathan Hseu008910f2017-08-25 14:01:05 -0700189 if os.path.isfile(python_bin_path) and os.access(
190 python_bin_path, os.X_OK):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700191 break
192 elif not os.path.exists(python_bin_path):
193 print('Invalid python path: %s cannot be found.' % python_bin_path)
194 else:
195 print('%s is not executable. Is it the python binary?' % python_bin_path)
196 environ_cp['PYTHON_BIN_PATH'] = ''
197
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700198 # Convert python path to Windows style before checking lib and version
Martin Wicked57572e2017-09-02 19:21:45 -0700199 if is_windows() or is_cygwin():
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700200 python_bin_path = cygpath(python_bin_path)
201
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700202 # Get PYTHON_LIB_PATH
203 python_lib_path = environ_cp.get('PYTHON_LIB_PATH')
204 if not python_lib_path:
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700205 python_lib_paths = get_python_path(environ_cp, python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700206 if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1':
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700207 python_lib_path = python_lib_paths[0]
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700208 else:
Jonathan Hseu008910f2017-08-25 14:01:05 -0700209 print('Found possible Python library paths:\n %s' %
210 '\n '.join(python_lib_paths))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700211 default_python_lib_path = python_lib_paths[0]
212 python_lib_path = get_input(
Jonathan Hseu008910f2017-08-25 14:01:05 -0700213 'Please input the desired Python library path to use. '
214 'Default is [%s]\n' % python_lib_paths[0])
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700215 if not python_lib_path:
216 python_lib_path = default_python_lib_path
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700217 environ_cp['PYTHON_LIB_PATH'] = python_lib_path
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700218
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700219 python_major_version = get_python_major_version(python_bin_path)
220
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700221 # Convert python path to Windows style before writing into bazel.rc
Martin Wicked57572e2017-09-02 19:21:45 -0700222 if is_windows() or is_cygwin():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700223 python_lib_path = cygpath(python_lib_path)
224
225 # Set-up env variables used by python_configure.bzl
226 write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path)
227 write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700228 write_to_bazelrc('build --force_python=py%s' % python_major_version)
229 write_to_bazelrc('build --host_force_python=py%s' % python_major_version)
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -0700230 write_to_bazelrc('build --python_path=\"%s"' % python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700231 environ_cp['PYTHON_BIN_PATH'] = python_bin_path
232
233 # Write tools/python_bin_path.sh
Shanqing Cai71445712018-03-12 19:33:52 -0700234 with open(os.path.join(
235 _TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'), 'w') as f:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700236 f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path)
237
238
Shanqing Cai71445712018-03-12 19:33:52 -0700239def reset_tf_configure_bazelrc(workspace_path):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700240 """Reset file that contains customized config settings."""
241 open(_TF_BAZELRC, 'w').close()
Shanqing Cai71445712018-03-12 19:33:52 -0700242 bazelrc_path = os.path.join(workspace_path, '.bazelrc')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700243
Shanqing Cai71445712018-03-12 19:33:52 -0700244 data = []
245 if os.path.exists(bazelrc_path):
246 with open(bazelrc_path, 'r') as f:
247 data = f.read().splitlines()
248 with open(bazelrc_path, 'w') as f:
249 for l in data:
250 if _TF_BAZELRC_FILENAME in l:
251 continue
252 f.write('%s\n' % l)
253 if is_windows():
254 tf_bazelrc_path = _TF_BAZELRC.replace("\\", "/")
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700255 else:
Shanqing Cai71445712018-03-12 19:33:52 -0700256 tf_bazelrc_path = _TF_BAZELRC
257 f.write('import %s\n' % tf_bazelrc_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700258
259
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700260def cleanup_makefile():
261 """Delete any leftover BUILD files from the Makefile build.
262
263 These files could interfere with Bazel parsing.
264 """
Shanqing Cai71445712018-03-12 19:33:52 -0700265 makefile_download_dir = os.path.join(
266 _TF_WORKSPACE_ROOT, 'tensorflow', 'contrib', 'makefile', 'downloads')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700267 if os.path.isdir(makefile_download_dir):
268 for root, _, filenames in os.walk(makefile_download_dir):
269 for f in filenames:
270 if f.endswith('BUILD'):
271 os.remove(os.path.join(root, f))
272
273
274def get_var(environ_cp,
275 var_name,
276 query_item,
277 enabled_by_default,
278 question=None,
279 yes_reply=None,
280 no_reply=None):
281 """Get boolean input from user.
282
283 If var_name is not set in env, ask user to enable query_item or not. If the
284 response is empty, use the default.
285
286 Args:
287 environ_cp: copy of the os.environ.
288 var_name: string for name of environment variable, e.g. "TF_NEED_HDFS".
289 query_item: string for feature related to the variable, e.g. "Hadoop File
290 System".
291 enabled_by_default: boolean for default behavior.
292 question: optional string for how to ask for user input.
Michael Cased90054e2018-02-07 14:36:00 -0800293 yes_reply: optional string for reply when feature is enabled.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700294 no_reply: optional string for reply when feature is disabled.
295
296 Returns:
297 boolean value of the variable.
Frank Chenc4ef9272018-01-10 11:36:52 -0800298
299 Raises:
300 UserInputError: if an environment variable is set, but it cannot be
301 interpreted as a boolean indicator, assume that the user has made a
302 scripting error, and will continue to provide invalid input.
303 Raise the error to avoid infinitely looping.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700304 """
305 if not question:
306 question = 'Do you wish to build TensorFlow with %s support?' % query_item
307 if not yes_reply:
308 yes_reply = '%s support will be enabled for TensorFlow.' % query_item
309 if not no_reply:
310 no_reply = 'No %s' % yes_reply
311
312 yes_reply += '\n'
313 no_reply += '\n'
314
315 if enabled_by_default:
316 question += ' [Y/n]: '
317 else:
318 question += ' [y/N]: '
319
320 var = environ_cp.get(var_name)
Frank Chenc4ef9272018-01-10 11:36:52 -0800321 if var is not None:
322 var_content = var.strip().lower()
323 true_strings = ('1', 't', 'true', 'y', 'yes')
324 false_strings = ('0', 'f', 'false', 'n', 'no')
325 if var_content in true_strings:
326 var = True
327 elif var_content in false_strings:
328 var = False
329 else:
330 raise UserInputError(
331 'Environment variable %s must be set as a boolean indicator.\n'
332 'The following are accepted as TRUE : %s.\n'
333 'The following are accepted as FALSE: %s.\n'
334 'Current value is %s.' % (
335 var_name, ', '.join(true_strings), ', '.join(false_strings),
336 var))
337
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700338 while var is None:
339 user_input_origin = get_input(question)
340 user_input = user_input_origin.strip().lower()
341 if user_input == 'y':
342 print(yes_reply)
343 var = True
344 elif user_input == 'n':
345 print(no_reply)
346 var = False
347 elif not user_input:
348 if enabled_by_default:
349 print(yes_reply)
350 var = True
351 else:
352 print(no_reply)
353 var = False
354 else:
355 print('Invalid selection: %s' % user_input_origin)
356 return var
357
358
359def set_build_var(environ_cp, var_name, query_item, option_name,
Michael Case98850a52017-09-14 13:35:57 -0700360 enabled_by_default, bazel_config_name=None):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700361 """Set if query_item will be enabled for the build.
362
363 Ask user if query_item will be enabled. Default is used if no input is given.
364 Set subprocess environment variable and write to .bazelrc if enabled.
365
366 Args:
367 environ_cp: copy of the os.environ.
368 var_name: string for name of environment variable, e.g. "TF_NEED_HDFS".
369 query_item: string for feature related to the variable, e.g. "Hadoop File
370 System".
371 option_name: string for option to define in .bazelrc.
372 enabled_by_default: boolean for default behavior.
Michael Case98850a52017-09-14 13:35:57 -0700373 bazel_config_name: Name for Bazel --config argument to enable build feature.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700374 """
375
376 var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default)))
377 environ_cp[var_name] = var
378 if var == '1':
379 write_to_bazelrc('build --define %s=true' % option_name)
Michael Case98850a52017-09-14 13:35:57 -0700380 elif bazel_config_name is not None:
381 # TODO(mikecase): Migrate all users of configure.py to use --config Bazel
382 # options and not to set build configs through environment variables.
383 write_to_bazelrc('build:%s --define %s=true'
384 % (bazel_config_name, option_name))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700385
386
387def set_action_env_var(environ_cp,
388 var_name,
389 query_item,
390 enabled_by_default,
391 question=None,
392 yes_reply=None,
393 no_reply=None):
394 """Set boolean action_env variable.
395
396 Ask user if query_item will be enabled. Default is used if no input is given.
397 Set environment variable and write to .bazelrc.
398
399 Args:
400 environ_cp: copy of the os.environ.
401 var_name: string for name of environment variable, e.g. "TF_NEED_HDFS".
402 query_item: string for feature related to the variable, e.g. "Hadoop File
403 System".
404 enabled_by_default: boolean for default behavior.
405 question: optional string for how to ask for user input.
Michael Cased90054e2018-02-07 14:36:00 -0800406 yes_reply: optional string for reply when feature is enabled.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700407 no_reply: optional string for reply when feature is disabled.
408 """
409 var = int(
410 get_var(environ_cp, var_name, query_item, enabled_by_default, question,
411 yes_reply, no_reply))
412
413 write_action_env_to_bazelrc(var_name, var)
414 environ_cp[var_name] = str(var)
415
416
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700417def convert_version_to_int(version):
418 """Convert a version number to a integer that can be used to compare.
419
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700420 Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The
421 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored.
422
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700423 Args:
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700424 version: a version to be converted
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700425
426 Returns:
427 An integer if converted successfully, otherwise return None.
428 """
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700429 version = version.split('-')[0]
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700430 version_segments = version.split('.')
431 for seg in version_segments:
432 if not seg.isdigit():
433 return None
434
435 version_str = ''.join(['%03d' % int(seg) for seg in version_segments])
436 return int(version_str)
437
438
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700439def check_bazel_version(min_version):
Yifei Fengdce9a492018-02-22 14:24:57 -0800440 """Check installed bazel version is at least min_version.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700441
442 Args:
443 min_version: string for minimum bazel version.
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700444
445 Returns:
446 The bazel version detected.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700447 """
Jonathan Hseu008910f2017-08-25 14:01:05 -0700448 if which('bazel') is None:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700449 print('Cannot find bazel. Please install bazel.')
450 sys.exit(0)
Shanqing Cai71445712018-03-12 19:33:52 -0700451 curr_version = run_shell(['bazel', '--batch', '--bazelrc=/dev/null', 'version'])
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700452
453 for line in curr_version.split('\n'):
454 if 'Build label: ' in line:
455 curr_version = line.split('Build label: ')[1]
456 break
457
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700458 min_version_int = convert_version_to_int(min_version)
459 curr_version_int = convert_version_to_int(curr_version)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700460
461 # Check if current bazel version can be detected properly.
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700462 if not curr_version_int:
463 print('WARNING: current bazel installation is not a release version.')
464 print('Make sure you are running at least bazel %s' % min_version)
465 return curr_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700466
Michael Cased94271a2017-08-22 17:26:52 -0700467 print('You have bazel %s installed.' % curr_version)
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700468
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700469 if curr_version_int < min_version_int:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700470 print('Please upgrade your bazel installation to version %s or higher to '
471 'build TensorFlow!' % min_version)
472 sys.exit(0)
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700473 return curr_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700474
475
476def set_cc_opt_flags(environ_cp):
477 """Set up architecture-dependent optimization flags.
478
479 Also append CC optimization flags to bazel.rc..
480
481 Args:
482 environ_cp: copy of the os.environ.
483 """
484 if is_ppc64le():
485 # gcc on ppc64le does not support -march, use mcpu instead
486 default_cc_opt_flags = '-mcpu=native'
487 else:
488 default_cc_opt_flags = '-march=native'
489 question = ('Please specify optimization flags to use during compilation when'
490 ' bazel option "--config=opt" is specified [Default is %s]: '
491 ) % default_cc_opt_flags
492 cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS',
493 question, default_cc_opt_flags)
494 for opt in cc_opt_flags.split():
Michael Case00177422017-11-10 13:14:03 -0800495 write_to_bazelrc('build:opt --copt=%s' % opt)
496 # It should be safe on the same build host.
Shanqing Cai71445712018-03-12 19:33:52 -0700497 if not is_ppc64le():
498 write_to_bazelrc('build:opt --host_copt=-march=native')
Michael Casebb3355d2017-11-09 08:46:31 -0800499 write_to_bazelrc('build:opt --define with_default_optimizations=true')
Michael Case00177422017-11-10 13:14:03 -0800500 # TODO(mikecase): Remove these default defines once we are able to get
501 # TF Lite targets building without them.
Andrew Selle0b154392017-11-10 10:35:35 -0800502 write_to_bazelrc('build --copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK')
503 write_to_bazelrc('build --host_copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700504
505
506def set_tf_cuda_clang(environ_cp):
507 """set TF_CUDA_CLANG action_env.
508
509 Args:
510 environ_cp: copy of the os.environ.
511 """
512 question = 'Do you want to use clang as CUDA compiler?'
513 yes_reply = 'Clang will be used as CUDA compiler.'
514 no_reply = 'nvcc will be used as CUDA compiler.'
515 set_action_env_var(
516 environ_cp,
517 'TF_CUDA_CLANG',
518 None,
519 False,
520 question=question,
521 yes_reply=yes_reply,
522 no_reply=no_reply)
523
524
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800525def set_tf_download_clang(environ_cp):
526 """Set TF_DOWNLOAD_CLANG action_env."""
Ilya Biryukov9e651e42018-03-22 05:33:42 -0700527 question = 'Do you wish to download a fresh release of clang? (Experimental)'
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800528 yes_reply = 'Clang will be downloaded and used to compile tensorflow.'
529 no_reply = 'Clang will not be downloaded.'
530 set_action_env_var(
531 environ_cp,
532 'TF_DOWNLOAD_CLANG',
533 None,
534 False,
535 question=question,
536 yes_reply=yes_reply,
537 no_reply=no_reply)
538
539
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700540def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var,
541 var_default):
542 """Get var_name either from env, or user or default.
543
544 If var_name has been set as environment variable, use the preset value, else
545 ask for user input. If no input is provided, the default is used.
546
547 Args:
548 environ_cp: copy of the os.environ.
549 var_name: string for name of environment variable, e.g. "TF_NEED_HDFS".
550 ask_for_var: string for how to ask for user input.
551 var_default: default value string.
552
553 Returns:
554 string value for var_name
555 """
556 var = environ_cp.get(var_name)
557 if not var:
558 var = get_input(ask_for_var)
Michael Cased94271a2017-08-22 17:26:52 -0700559 print('\n')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700560 if not var:
561 var = var_default
562 return var
563
564
565def set_clang_cuda_compiler_path(environ_cp):
566 """Set CLANG_CUDA_COMPILER_PATH."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700567 default_clang_path = which('clang') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700568 ask_clang_path = ('Please specify which clang should be used as device and '
569 'host compiler. [Default is %s]: ') % default_clang_path
570
571 while True:
572 clang_cuda_compiler_path = get_from_env_or_user_or_default(
573 environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path,
574 default_clang_path)
575 if os.path.exists(clang_cuda_compiler_path):
576 break
577
578 # Reset and retry
579 print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path)
580 environ_cp['CLANG_CUDA_COMPILER_PATH'] = ''
581
582 # Set CLANG_CUDA_COMPILER_PATH
583 environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path
584 write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH',
585 clang_cuda_compiler_path)
586
587
Austin Anderson6afface2017-12-05 11:59:17 -0800588def prompt_loop_or_load_from_env(
589 environ_cp,
590 var_name,
591 var_default,
592 ask_for_var,
593 check_success,
594 error_msg,
595 suppress_default_error=False,
596 n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS
597):
598 """Loop over user prompts for an ENV param until receiving a valid response.
599
600 For the env param var_name, read from the environment or verify user input
601 until receiving valid input. When done, set var_name in the environ_cp to its
602 new value.
603
604 Args:
605 environ_cp: (Dict) copy of the os.environ.
606 var_name: (String) string for name of environment variable, e.g. "TF_MYVAR".
607 var_default: (String) default value string.
608 ask_for_var: (String) string for how to ask for user input.
609 check_success: (Function) function that takes one argument and returns a
610 boolean. Should return True if the value provided is considered valid. May
611 contain a complex error message if error_msg does not provide enough
612 information. In that case, set suppress_default_error to True.
613 error_msg: (String) String with one and only one '%s'. Formatted with each
614 invalid response upon check_success(input) failure.
615 suppress_default_error: (Bool) Suppress the above error message in favor of
616 one from the check_success function.
617 n_ask_attempts: (Integer) Number of times to query for valid input before
618 raising an error and quitting.
619
620 Returns:
621 [String] The value of var_name after querying for input.
622
623 Raises:
624 UserInputError: if a query has been attempted n_ask_attempts times without
Frank Chenc4ef9272018-01-10 11:36:52 -0800625 success, assume that the user has made a scripting error, and will
626 continue to provide invalid input. Raise the error to avoid infinitely
627 looping.
Austin Anderson6afface2017-12-05 11:59:17 -0800628 """
629 default = environ_cp.get(var_name) or var_default
630 full_query = '%s [Default is %s]: ' % (
631 ask_for_var,
632 default,
633 )
634
635 for _ in range(n_ask_attempts):
636 val = get_from_env_or_user_or_default(environ_cp,
637 var_name,
638 full_query,
639 default)
640 if check_success(val):
641 break
642 if not suppress_default_error:
643 print(error_msg % val)
644 environ_cp[var_name] = ''
645 else:
646 raise UserInputError('Invalid %s setting was provided %d times in a row. '
647 'Assuming to be a scripting mistake.' %
648 (var_name, n_ask_attempts))
649
650 environ_cp[var_name] = val
651 return val
652
653
654def create_android_ndk_rule(environ_cp):
655 """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule."""
656 if is_windows() or is_cygwin():
657 default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' %
658 environ_cp['APPDATA'])
659 elif is_macos():
660 default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME']
661 else:
662 default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME']
663
664 def valid_ndk_path(path):
665 return (os.path.exists(path) and
666 os.path.exists(os.path.join(path, 'source.properties')))
667
668 android_ndk_home_path = prompt_loop_or_load_from_env(
669 environ_cp,
670 var_name='ANDROID_NDK_HOME',
671 var_default=default_ndk_path,
672 ask_for_var='Please specify the home path of the Android NDK to use.',
673 check_success=valid_ndk_path,
674 error_msg=('The path %s or its child file "source.properties" '
675 'does not exist.')
676 )
677
678 write_android_ndk_workspace_rule(android_ndk_home_path)
679
680
681def create_android_sdk_rule(environ_cp):
682 """Set Android variables and write Android SDK WORKSPACE rule."""
683 if is_windows() or is_cygwin():
684 default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA'])
685 elif is_macos():
686 default_sdk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME']
687 else:
688 default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME']
689
690 def valid_sdk_path(path):
691 return (os.path.exists(path) and
692 os.path.exists(os.path.join(path, 'platforms')) and
693 os.path.exists(os.path.join(path, 'build-tools')))
694
695 android_sdk_home_path = prompt_loop_or_load_from_env(
696 environ_cp,
697 var_name='ANDROID_SDK_HOME',
698 var_default=default_sdk_path,
699 ask_for_var='Please specify the home path of the Android SDK to use.',
700 check_success=valid_sdk_path,
701 error_msg=('Either %s does not exist, or it does not contain the '
702 'subdirectories "platforms" and "build-tools".'))
703
704 platforms = os.path.join(android_sdk_home_path, 'platforms')
705 api_levels = sorted(os.listdir(platforms))
706 api_levels = [x.replace('android-', '') for x in api_levels]
707
708 def valid_api_level(api_level):
709 return os.path.exists(os.path.join(android_sdk_home_path,
710 'platforms',
711 'android-' + api_level))
712
713 android_api_level = prompt_loop_or_load_from_env(
714 environ_cp,
715 var_name='ANDROID_API_LEVEL',
716 var_default=api_levels[-1],
717 ask_for_var=('Please specify the Android SDK API level to use. '
718 '[Available levels: %s]') % api_levels,
719 check_success=valid_api_level,
720 error_msg='Android-%s is not present in the SDK path.')
721
722 build_tools = os.path.join(android_sdk_home_path, 'build-tools')
723 versions = sorted(os.listdir(build_tools))
724
725 def valid_build_tools(version):
726 return os.path.exists(os.path.join(android_sdk_home_path,
727 'build-tools',
728 version))
729
730 android_build_tools_version = prompt_loop_or_load_from_env(
731 environ_cp,
732 var_name='ANDROID_BUILD_TOOLS_VERSION',
733 var_default=versions[-1],
734 ask_for_var=('Please specify an Android build tools version to use. '
735 '[Available versions: %s]') % versions,
736 check_success=valid_build_tools,
737 error_msg=('The selected SDK does not have build-tools version %s '
738 'available.'))
739
740 write_android_sdk_workspace_rule(android_sdk_home_path,
741 android_build_tools_version,
742 android_api_level)
743
744
745def write_android_sdk_workspace_rule(android_sdk_home_path,
746 android_build_tools_version,
747 android_api_level):
748 print('Writing android_sdk_workspace rule.\n')
749 with open(_TF_WORKSPACE, 'a') as f:
750 f.write("""
751android_sdk_repository(
752 name="androidsdk",
753 api_level=%s,
754 path="%s",
755 build_tools_version="%s")\n
756""" % (android_api_level, android_sdk_home_path, android_build_tools_version))
757
758
759def write_android_ndk_workspace_rule(android_ndk_home_path):
760 print('Writing android_ndk_workspace rule.')
761 ndk_api_level = check_ndk_level(android_ndk_home_path)
762 if int(ndk_api_level) not in _SUPPORTED_ANDROID_NDK_VERSIONS:
763 print('WARNING: The API level of the NDK in %s is %s, which is not '
764 'supported by Bazel (officially supported versions: %s). Please use '
765 'another version. Compiling Android targets may result in confusing '
766 'errors.\n' % (android_ndk_home_path, ndk_api_level,
767 _SUPPORTED_ANDROID_NDK_VERSIONS))
768 with open(_TF_WORKSPACE, 'a') as f:
769 f.write("""
770android_ndk_repository(
771 name="androidndk",
772 path="%s",
773 api_level=%s)\n
774""" % (android_ndk_home_path, ndk_api_level))
775
776
777def check_ndk_level(android_ndk_home_path):
778 """Check the revision number of an Android NDK path."""
779 properties_path = '%s/source.properties' % android_ndk_home_path
780 if is_windows() or is_cygwin():
781 properties_path = cygpath(properties_path)
782 with open(properties_path, 'r') as f:
783 filedata = f.read()
784
785 revision = re.search(r'Pkg.Revision = (\d+)', filedata)
786 if revision:
787 return revision.group(1)
788 return None
789
790
791def workspace_has_any_android_rule():
792 """Check the WORKSPACE for existing android_*_repository rules."""
793 with open(_TF_WORKSPACE, 'r') as f:
794 workspace = f.read()
795 has_any_rule = re.search(r'^android_[ns]dk_repository',
796 workspace,
797 re.MULTILINE)
798 return has_any_rule
799
800
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700801def set_gcc_host_compiler_path(environ_cp):
802 """Set GCC_HOST_COMPILER_PATH."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700803 default_gcc_host_compiler_path = which('gcc') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700804 cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH')
805
806 if os.path.islink(cuda_bin_symlink):
807 # os.readlink is only available in linux
Jonathan Hseu008910f2017-08-25 14:01:05 -0700808 default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700809
Austin Anderson6afface2017-12-05 11:59:17 -0800810 gcc_host_compiler_path = prompt_loop_or_load_from_env(
811 environ_cp,
812 var_name='GCC_HOST_COMPILER_PATH',
813 var_default=default_gcc_host_compiler_path,
814 ask_for_var=
815 'Please specify which gcc should be used by nvcc as the host compiler.',
816 check_success=os.path.exists,
817 error_msg='Invalid gcc path. %s cannot be found.',
818 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700819
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700820 write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path)
821
822
Ankur Taly0e6f39d2018-02-16 18:22:55 -0800823def reformat_version_sequence(version_str, sequence_count):
824 """Reformat the version string to have the given number of sequences.
825
826 For example:
827 Given (7, 2) -> 7.0
828 (7.0.1, 2) -> 7.0
829 (5, 1) -> 5
830 (5.0.3.2, 1) -> 5
831
832 Args:
833 version_str: String, the version string.
834 sequence_count: int, an integer.
835 Returns:
836 string, reformatted version string.
837 """
838 v = version_str.split('.')
839 if len(v) < sequence_count:
840 v = v + (['0'] * (sequence_count - len(v)))
841
842 return '.'.join(v[:sequence_count])
843
844
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700845def set_tf_cuda_version(environ_cp):
846 """Set CUDA_TOOLKIT_PATH and TF_CUDA_VERSION."""
847 ask_cuda_version = (
848 'Please specify the CUDA SDK version you want to use, '
849 'e.g. 7.0. [Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION
850
Austin Andersonf9a88f82017-12-13 11:49:40 -0800851 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700852 # Configure the Cuda SDK version to use.
853 tf_cuda_version = get_from_env_or_user_or_default(
854 environ_cp, 'TF_CUDA_VERSION', ask_cuda_version, _DEFAULT_CUDA_VERSION)
Ankur Taly0e6f39d2018-02-16 18:22:55 -0800855 tf_cuda_version = reformat_version_sequence(str(tf_cuda_version), 2)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700856
857 # Find out where the CUDA toolkit is installed
858 default_cuda_path = _DEFAULT_CUDA_PATH
Martin Wicked57572e2017-09-02 19:21:45 -0700859 if is_windows() or is_cygwin():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700860 default_cuda_path = cygpath(
861 environ_cp.get('CUDA_PATH', _DEFAULT_CUDA_PATH_WIN))
862 elif is_linux():
863 # If the default doesn't exist, try an alternative default.
864 if (not os.path.exists(default_cuda_path)
865 ) and os.path.exists(_DEFAULT_CUDA_PATH_LINUX):
866 default_cuda_path = _DEFAULT_CUDA_PATH_LINUX
867 ask_cuda_path = ('Please specify the location where CUDA %s toolkit is'
868 ' installed. Refer to README.md for more details. '
869 '[Default is %s]: ') % (tf_cuda_version, default_cuda_path)
870 cuda_toolkit_path = get_from_env_or_user_or_default(
871 environ_cp, 'CUDA_TOOLKIT_PATH', ask_cuda_path, default_cuda_path)
872
873 if is_windows():
874 cuda_rt_lib_path = 'lib/x64/cudart.lib'
875 elif is_linux():
876 cuda_rt_lib_path = 'lib64/libcudart.so.%s' % tf_cuda_version
877 elif is_macos():
878 cuda_rt_lib_path = 'lib/libcudart.%s.dylib' % tf_cuda_version
879
880 cuda_toolkit_path_full = os.path.join(cuda_toolkit_path, cuda_rt_lib_path)
881 if os.path.exists(cuda_toolkit_path_full):
882 break
883
884 # Reset and retry
885 print('Invalid path to CUDA %s toolkit. %s cannot be found' %
886 (tf_cuda_version, cuda_toolkit_path_full))
887 environ_cp['TF_CUDA_VERSION'] = ''
888 environ_cp['CUDA_TOOLKIT_PATH'] = ''
889
Austin Andersonf9a88f82017-12-13 11:49:40 -0800890 else:
891 raise UserInputError('Invalid TF_CUDA_SETTING setting was provided %d '
892 'times in a row. Assuming to be a scripting mistake.' %
893 _DEFAULT_PROMPT_ASK_ATTEMPTS)
894
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700895 # Set CUDA_TOOLKIT_PATH and TF_CUDA_VERSION
896 environ_cp['CUDA_TOOLKIT_PATH'] = cuda_toolkit_path
897 write_action_env_to_bazelrc('CUDA_TOOLKIT_PATH', cuda_toolkit_path)
898 environ_cp['TF_CUDA_VERSION'] = tf_cuda_version
899 write_action_env_to_bazelrc('TF_CUDA_VERSION', tf_cuda_version)
900
901
Yifei Fengb1d8c592017-11-22 13:42:21 -0800902def set_tf_cudnn_version(environ_cp):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700903 """Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION."""
904 ask_cudnn_version = (
Jonathan Hseu008910f2017-08-25 14:01:05 -0700905 'Please specify the cuDNN version you want to use. '
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700906 '[Leave empty to default to cuDNN %s.0]: ') % _DEFAULT_CUDNN_VERSION
907
Austin Andersonf9a88f82017-12-13 11:49:40 -0800908 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700909 tf_cudnn_version = get_from_env_or_user_or_default(
910 environ_cp, 'TF_CUDNN_VERSION', ask_cudnn_version,
911 _DEFAULT_CUDNN_VERSION)
Ankur Taly0e6f39d2018-02-16 18:22:55 -0800912 tf_cudnn_version = reformat_version_sequence(str(tf_cudnn_version), 1)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700913
914 default_cudnn_path = environ_cp.get('CUDA_TOOLKIT_PATH')
915 ask_cudnn_path = (r'Please specify the location where cuDNN %s library is '
916 'installed. Refer to README.md for more details. [Default'
917 ' is %s]:') % (tf_cudnn_version, default_cudnn_path)
918 cudnn_install_path = get_from_env_or_user_or_default(
919 environ_cp, 'CUDNN_INSTALL_PATH', ask_cudnn_path, default_cudnn_path)
920
921 # Result returned from "read" will be used unexpanded. That make "~"
922 # unusable. Going through one more level of expansion to handle that.
923 cudnn_install_path = os.path.realpath(
924 os.path.expanduser(cudnn_install_path))
Martin Wicked57572e2017-09-02 19:21:45 -0700925 if is_windows() or is_cygwin():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700926 cudnn_install_path = cygpath(cudnn_install_path)
927
928 if is_windows():
929 cuda_dnn_lib_path = 'lib/x64/cudnn.lib'
930 cuda_dnn_lib_alt_path = 'lib/x64/cudnn.lib'
931 elif is_linux():
932 cuda_dnn_lib_path = 'lib64/libcudnn.so.%s' % tf_cudnn_version
933 cuda_dnn_lib_alt_path = 'libcudnn.so.%s' % tf_cudnn_version
934 elif is_macos():
935 cuda_dnn_lib_path = 'lib/libcudnn.%s.dylib' % tf_cudnn_version
936 cuda_dnn_lib_alt_path = 'libcudnn.%s.dylib' % tf_cudnn_version
937
938 cuda_dnn_lib_path_full = os.path.join(cudnn_install_path, cuda_dnn_lib_path)
939 cuda_dnn_lib_alt_path_full = os.path.join(cudnn_install_path,
940 cuda_dnn_lib_alt_path)
941 if os.path.exists(cuda_dnn_lib_path_full) or os.path.exists(
942 cuda_dnn_lib_alt_path_full):
943 break
944
945 # Try another alternative for Linux
946 if is_linux():
Jonathan Hseu008910f2017-08-25 14:01:05 -0700947 ldconfig_bin = which('ldconfig') or '/sbin/ldconfig'
948 cudnn_path_from_ldconfig = run_shell([ldconfig_bin, '-p'])
949 cudnn_path_from_ldconfig = re.search('.*libcudnn.so .* => (.*)',
A. Unique TensorFlowere7223582017-09-06 17:57:04 -0700950 cudnn_path_from_ldconfig)
951 if cudnn_path_from_ldconfig:
952 cudnn_path_from_ldconfig = cudnn_path_from_ldconfig.group(1)
953 if os.path.exists('%s.%s' % (cudnn_path_from_ldconfig,
954 tf_cudnn_version)):
955 cudnn_install_path = os.path.dirname(cudnn_path_from_ldconfig)
956 break
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700957
958 # Reset and Retry
959 print(
960 'Invalid path to cuDNN %s toolkit. None of the following files can be '
961 'found:' % tf_cudnn_version)
962 print(cuda_dnn_lib_path_full)
963 print(cuda_dnn_lib_alt_path_full)
964 if is_linux():
965 print('%s.%s' % (cudnn_path_from_ldconfig, tf_cudnn_version))
966
967 environ_cp['TF_CUDNN_VERSION'] = ''
Austin Andersonf9a88f82017-12-13 11:49:40 -0800968 else:
969 raise UserInputError('Invalid TF_CUDNN setting was provided %d '
970 'times in a row. Assuming to be a scripting mistake.' %
971 _DEFAULT_PROMPT_ASK_ATTEMPTS)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700972
973 # Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION
974 environ_cp['CUDNN_INSTALL_PATH'] = cudnn_install_path
975 write_action_env_to_bazelrc('CUDNN_INSTALL_PATH', cudnn_install_path)
976 environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version
977 write_action_env_to_bazelrc('TF_CUDNN_VERSION', tf_cudnn_version)
978
979
Guangda Lai76f69382018-01-25 23:59:19 -0800980def set_tf_tensorrt_install_path(environ_cp):
981 """Set TENSORRT_INSTALL_PATH and TF_TENSORRT_VERSION.
982
983 Adapted from code contributed by Sami Kama (https://github.com/samikama).
984
985 Args:
986 environ_cp: copy of the os.environ.
987
988 Raises:
989 ValueError: if this method was called under non-Linux platform.
990 UserInputError: if user has provided invalid input multiple times.
991 """
992 if not is_linux():
993 raise ValueError('Currently TensorRT is only supported on Linux platform.')
994
995 # Ask user whether to add TensorRT support.
996 if str(int(get_var(
997 environ_cp, 'TF_NEED_TENSORRT', 'TensorRT', False))) != '1':
998 return
999
1000 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS):
1001 ask_tensorrt_path = (r'Please specify the location where TensorRT is '
1002 'installed. [Default is %s]:') % (
1003 _DEFAULT_TENSORRT_PATH_LINUX)
1004 trt_install_path = get_from_env_or_user_or_default(
1005 environ_cp, 'TENSORRT_INSTALL_PATH', ask_tensorrt_path,
1006 _DEFAULT_TENSORRT_PATH_LINUX)
1007
1008 # Result returned from "read" will be used unexpanded. That make "~"
1009 # unusable. Going through one more level of expansion to handle that.
1010 trt_install_path = os.path.realpath(
1011 os.path.expanduser(trt_install_path))
1012
1013 def find_libs(search_path):
1014 """Search for libnvinfer.so in "search_path"."""
1015 fl = set()
1016 if os.path.exists(search_path) and os.path.isdir(search_path):
1017 fl.update([os.path.realpath(os.path.join(search_path, x))
1018 for x in os.listdir(search_path) if 'libnvinfer.so' in x])
1019 return fl
1020
1021 possible_files = find_libs(trt_install_path)
1022 possible_files.update(find_libs(os.path.join(trt_install_path, 'lib')))
1023 possible_files.update(find_libs(os.path.join(trt_install_path, 'lib64')))
1024
1025 def is_compatible(tensorrt_lib, cuda_ver, cudnn_ver):
1026 """Check the compatibility between tensorrt and cudnn/cudart libraries."""
1027 ldd_bin = which('ldd') or '/usr/bin/ldd'
1028 ldd_out = run_shell([ldd_bin, tensorrt_lib]).split(os.linesep)
1029 cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$')
1030 cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$')
1031 cudnn = None
1032 cudart = None
1033 for line in ldd_out:
1034 if 'libcudnn.so' in line:
1035 cudnn = cudnn_pattern.search(line)
1036 elif 'libcudart.so' in line:
1037 cudart = cuda_pattern.search(line)
1038 if cudnn and len(cudnn.group(1)):
1039 cudnn = convert_version_to_int(cudnn.group(1))
1040 if cudart and len(cudart.group(1)):
1041 cudart = convert_version_to_int(cudart.group(1))
1042 return (cudnn == cudnn_ver) and (cudart == cuda_ver)
1043
1044 cuda_ver = convert_version_to_int(environ_cp['TF_CUDA_VERSION'])
1045 cudnn_ver = convert_version_to_int(environ_cp['TF_CUDNN_VERSION'])
1046 nvinfer_pattern = re.compile('.*libnvinfer.so.?(.*)$')
1047 highest_ver = [0, None, None]
1048
1049 for lib_file in possible_files:
1050 if is_compatible(lib_file, cuda_ver, cudnn_ver):
Jacques Pienaar2d0531d2018-03-21 12:07:51 -07001051 matches = nvinfer_pattern.search(lib_file)
1052 if len(matches.groups()) == 0:
1053 continue
1054 ver_str = matches.group(1)
Guangda Lai76f69382018-01-25 23:59:19 -08001055 ver = convert_version_to_int(ver_str) if len(ver_str) else 0
1056 if ver > highest_ver[0]:
1057 highest_ver = [ver, ver_str, lib_file]
1058 if highest_ver[1] is not None:
1059 trt_install_path = os.path.dirname(highest_ver[2])
1060 tf_tensorrt_version = highest_ver[1]
1061 break
1062
1063 # Try another alternative from ldconfig.
1064 ldconfig_bin = which('ldconfig') or '/sbin/ldconfig'
1065 ldconfig_output = run_shell([ldconfig_bin, '-p'])
1066 search_result = re.search(
1067 '.*libnvinfer.so\\.?([0-9.]*).* => (.*)', ldconfig_output)
1068 if search_result:
1069 libnvinfer_path_from_ldconfig = search_result.group(2)
1070 if os.path.exists(libnvinfer_path_from_ldconfig):
1071 if is_compatible(libnvinfer_path_from_ldconfig, cuda_ver, cudnn_ver):
1072 trt_install_path = os.path.dirname(libnvinfer_path_from_ldconfig)
1073 tf_tensorrt_version = search_result.group(1)
1074 break
1075
1076 # Reset and Retry
Yifei Fengdce9a492018-02-22 14:24:57 -08001077 if possible_files:
1078 print('TensorRT libraries found in one the following directories',
1079 'are not compatible with selected cuda and cudnn installations')
1080 print(trt_install_path)
1081 print(os.path.join(trt_install_path, 'lib'))
1082 print(os.path.join(trt_install_path, 'lib64'))
1083 if search_result:
1084 print(libnvinfer_path_from_ldconfig)
1085 else:
1086 print(
1087 'Invalid path to TensorRT. None of the following files can be found:')
1088 print(trt_install_path)
1089 print(os.path.join(trt_install_path, 'lib'))
1090 print(os.path.join(trt_install_path, 'lib64'))
1091 if search_result:
1092 print(libnvinfer_path_from_ldconfig)
Guangda Lai76f69382018-01-25 23:59:19 -08001093
1094 else:
1095 raise UserInputError('Invalid TF_TENSORRT setting was provided %d '
1096 'times in a row. Assuming to be a scripting mistake.' %
1097 _DEFAULT_PROMPT_ASK_ATTEMPTS)
1098
1099 # Set TENSORRT_INSTALL_PATH and TF_TENSORRT_VERSION
1100 environ_cp['TENSORRT_INSTALL_PATH'] = trt_install_path
1101 write_action_env_to_bazelrc('TENSORRT_INSTALL_PATH', trt_install_path)
1102 environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version
1103 write_action_env_to_bazelrc('TF_TENSORRT_VERSION', tf_tensorrt_version)
1104
1105
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001106def get_native_cuda_compute_capabilities(environ_cp):
1107 """Get native cuda compute capabilities.
1108
1109 Args:
1110 environ_cp: copy of the os.environ.
1111 Returns:
1112 string of native cuda compute capabilities, separated by comma.
1113 """
1114 device_query_bin = os.path.join(
1115 environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery')
Jonathan Hseu008910f2017-08-25 14:01:05 -07001116 if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK):
1117 try:
1118 output = run_shell(device_query_bin).split('\n')
1119 pattern = re.compile('[0-9]*\\.[0-9]*')
1120 output = [pattern.search(x) for x in output if 'Capability' in x]
1121 output = ','.join(x.group() for x in output if x is not None)
1122 except subprocess.CalledProcessError:
1123 output = ''
1124 else:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001125 output = ''
1126 return output
1127
1128
1129def set_tf_cuda_compute_capabilities(environ_cp):
1130 """Set TF_CUDA_COMPUTE_CAPABILITIES."""
1131 while True:
1132 native_cuda_compute_capabilities = get_native_cuda_compute_capabilities(
1133 environ_cp)
1134 if not native_cuda_compute_capabilities:
1135 default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES
1136 else:
1137 default_cuda_compute_capabilities = native_cuda_compute_capabilities
1138
1139 ask_cuda_compute_capabilities = (
1140 'Please specify a list of comma-separated '
1141 'Cuda compute capabilities you want to '
1142 'build with.\nYou can find the compute '
1143 'capability of your device at: '
1144 'https://developer.nvidia.com/cuda-gpus.\nPlease'
1145 ' note that each additional compute '
1146 'capability significantly increases your '
1147 'build time and binary size. [Default is: %s]' %
1148 default_cuda_compute_capabilities)
1149 tf_cuda_compute_capabilities = get_from_env_or_user_or_default(
1150 environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES',
1151 ask_cuda_compute_capabilities, default_cuda_compute_capabilities)
1152 # Check whether all capabilities from the input is valid
1153 all_valid = True
1154 for compute_capability in tf_cuda_compute_capabilities.split(','):
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001155 m = re.match('[0-9]+.[0-9]+', compute_capability)
1156 if not m:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001157 print('Invalid compute capability: ' % compute_capability)
1158 all_valid = False
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001159 else:
1160 ver = int(m.group(0).split('.')[0])
1161 if ver < 3:
1162 print('Only compute capabilities 3.0 or higher are supported.')
1163 all_valid = False
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001164
1165 if all_valid:
1166 break
1167
1168 # Reset and Retry
1169 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = ''
1170
1171 # Set TF_CUDA_COMPUTE_CAPABILITIES
1172 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities
1173 write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES',
1174 tf_cuda_compute_capabilities)
1175
1176
1177def set_other_cuda_vars(environ_cp):
1178 """Set other CUDA related variables."""
1179 if is_windows():
1180 # The following three variables are needed for MSVC toolchain configuration
1181 # in Bazel
1182 environ_cp['CUDA_PATH'] = environ_cp.get('CUDA_TOOLKIT_PATH')
1183 environ_cp['CUDA_COMPUTE_CAPABILITIES'] = environ_cp.get(
1184 'TF_CUDA_COMPUTE_CAPABILITIES')
1185 environ_cp['NO_WHOLE_ARCHIVE_OPTION'] = 1
1186 write_action_env_to_bazelrc('CUDA_PATH', environ_cp.get('CUDA_PATH'))
1187 write_action_env_to_bazelrc('CUDA_COMPUTE_CAPABILITIE',
1188 environ_cp.get('CUDA_COMPUTE_CAPABILITIE'))
1189 write_action_env_to_bazelrc('NO_WHOLE_ARCHIVE_OPTION',
1190 environ_cp.get('NO_WHOLE_ARCHIVE_OPTION'))
1191 write_to_bazelrc('build --config=win-cuda')
1192 write_to_bazelrc('test --config=win-cuda')
1193 else:
1194 # If CUDA is enabled, always use GPU during build and test.
1195 if environ_cp.get('TF_CUDA_CLANG') == '1':
1196 write_to_bazelrc('build --config=cuda_clang')
1197 write_to_bazelrc('test --config=cuda_clang')
1198 else:
1199 write_to_bazelrc('build --config=cuda')
1200 write_to_bazelrc('test --config=cuda')
1201
1202
1203def set_host_cxx_compiler(environ_cp):
1204 """Set HOST_CXX_COMPILER."""
Jonathan Hseu008910f2017-08-25 14:01:05 -07001205 default_cxx_host_compiler = which('g++') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001206
Austin Anderson6afface2017-12-05 11:59:17 -08001207 host_cxx_compiler = prompt_loop_or_load_from_env(
1208 environ_cp,
1209 var_name='HOST_CXX_COMPILER',
1210 var_default=default_cxx_host_compiler,
1211 ask_for_var=('Please specify which C++ compiler should be used as the '
1212 'host C++ compiler.'),
1213 check_success=os.path.exists,
1214 error_msg='Invalid C++ compiler path. %s cannot be found.',
1215 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001216
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001217 write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler)
1218
1219
1220def set_host_c_compiler(environ_cp):
1221 """Set HOST_C_COMPILER."""
Jonathan Hseu008910f2017-08-25 14:01:05 -07001222 default_c_host_compiler = which('gcc') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001223
Austin Anderson6afface2017-12-05 11:59:17 -08001224 host_c_compiler = prompt_loop_or_load_from_env(
1225 environ_cp,
1226 var_name='HOST_C_COMPILER',
1227 var_default=default_c_host_compiler,
Shanqing Cai71445712018-03-12 19:33:52 -07001228 ask_for_var=('Please specify which C compiler should be used as the host '
Austin Anderson6afface2017-12-05 11:59:17 -08001229 'C compiler.'),
1230 check_success=os.path.exists,
1231 error_msg='Invalid C compiler path. %s cannot be found.',
1232 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001233
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001234 write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler)
1235
1236
1237def set_computecpp_toolkit_path(environ_cp):
1238 """Set COMPUTECPP_TOOLKIT_PATH."""
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001239
Austin Anderson6afface2017-12-05 11:59:17 -08001240 def toolkit_exists(toolkit_path):
1241 """Check if a computecpp toolkit path is valid."""
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001242 if is_linux():
1243 sycl_rt_lib_path = 'lib/libComputeCpp.so'
1244 else:
1245 sycl_rt_lib_path = ''
1246
Austin Anderson6afface2017-12-05 11:59:17 -08001247 sycl_rt_lib_path_full = os.path.join(toolkit_path,
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001248 sycl_rt_lib_path)
Austin Anderson6afface2017-12-05 11:59:17 -08001249 exists = os.path.exists(sycl_rt_lib_path_full)
1250 if not exists:
1251 print('Invalid SYCL %s library path. %s cannot be found' %
1252 (_TF_OPENCL_VERSION, sycl_rt_lib_path_full))
1253 return exists
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001254
Austin Anderson6afface2017-12-05 11:59:17 -08001255 computecpp_toolkit_path = prompt_loop_or_load_from_env(
1256 environ_cp,
1257 var_name='COMPUTECPP_TOOLKIT_PATH',
1258 var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH,
1259 ask_for_var=(
1260 'Please specify the location where ComputeCpp for SYCL %s is '
1261 'installed.' % _TF_OPENCL_VERSION),
1262 check_success=toolkit_exists,
1263 error_msg='Invalid SYCL compiler path. %s cannot be found.',
1264 suppress_default_error=True)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001265
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001266 write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH',
1267 computecpp_toolkit_path)
1268
Michael Cased31531a2018-01-05 14:09:41 -08001269
Dandelion Man?90e42f32017-12-15 18:15:07 -08001270def set_trisycl_include_dir(environ_cp):
Michael Cased31531a2018-01-05 14:09:41 -08001271 """Set TRISYCL_INCLUDE_DIR."""
Frank Chenc4ef9272018-01-10 11:36:52 -08001272
Dandelion Man?90e42f32017-12-15 18:15:07 -08001273 ask_trisycl_include_dir = ('Please specify the location of the triSYCL '
1274 'include directory. (Use --config=sycl_trisycl '
1275 'when building with Bazel) '
1276 '[Default is %s]: '
Michael Cased31531a2018-01-05 14:09:41 -08001277 ) % (_DEFAULT_TRISYCL_INCLUDE_DIR)
Frank Chenc4ef9272018-01-10 11:36:52 -08001278
Dandelion Man?90e42f32017-12-15 18:15:07 -08001279 while True:
1280 trisycl_include_dir = get_from_env_or_user_or_default(
Michael Cased31531a2018-01-05 14:09:41 -08001281 environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir,
1282 _DEFAULT_TRISYCL_INCLUDE_DIR)
Dandelion Man?90e42f32017-12-15 18:15:07 -08001283 if os.path.exists(trisycl_include_dir):
1284 break
1285
1286 print('Invalid triSYCL include directory, %s cannot be found'
1287 % (trisycl_include_dir))
1288
1289 # Set TRISYCL_INCLUDE_DIR
1290 environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir
1291 write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR',
1292 trisycl_include_dir)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001293
Yifei Fengb1d8c592017-11-22 13:42:21 -08001294
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001295def set_mpi_home(environ_cp):
1296 """Set MPI_HOME."""
Austin Anderson6afface2017-12-05 11:59:17 -08001297
Jonathan Hseu008910f2017-08-25 14:01:05 -07001298 default_mpi_home = which('mpirun') or which('mpiexec') or ''
1299 default_mpi_home = os.path.dirname(os.path.dirname(default_mpi_home))
1300
Austin Anderson6afface2017-12-05 11:59:17 -08001301 def valid_mpi_path(mpi_home):
1302 exists = (os.path.exists(os.path.join(mpi_home, 'include')) and
1303 os.path.exists(os.path.join(mpi_home, 'lib')))
1304 if not exists:
1305 print('Invalid path to the MPI Toolkit. %s or %s cannot be found' %
1306 (os.path.join(mpi_home, 'include'),
1307 os.path.exists(os.path.join(mpi_home, 'lib'))))
1308 return exists
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001309
Austin Anderson6afface2017-12-05 11:59:17 -08001310 _ = prompt_loop_or_load_from_env(
1311 environ_cp,
1312 var_name='MPI_HOME',
1313 var_default=default_mpi_home,
1314 ask_for_var='Please specify the MPI toolkit folder.',
1315 check_success=valid_mpi_path,
1316 error_msg='',
1317 suppress_default_error=True)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001318
1319
1320def set_other_mpi_vars(environ_cp):
1321 """Set other MPI related variables."""
1322 # Link the MPI header files
1323 mpi_home = environ_cp.get('MPI_HOME')
1324 symlink_force('%s/include/mpi.h' % mpi_home, 'third_party/mpi/mpi.h')
1325
1326 # Determine if we use OpenMPI or MVAPICH, these require different header files
1327 # to be included here to make bazel dependency checker happy
1328 if os.path.exists(os.path.join(mpi_home, 'include/mpi_portable_platform.h')):
1329 symlink_force(
1330 os.path.join(mpi_home, 'include/mpi_portable_platform.h'),
1331 'third_party/mpi/mpi_portable_platform.h')
1332 # TODO(gunan): avoid editing files in configure
1333 sed_in_place('third_party/mpi/mpi.bzl', 'MPI_LIB_IS_OPENMPI=False',
1334 'MPI_LIB_IS_OPENMPI=True')
1335 else:
1336 # MVAPICH / MPICH
1337 symlink_force(
1338 os.path.join(mpi_home, 'include/mpio.h'), 'third_party/mpi/mpio.h')
1339 symlink_force(
1340 os.path.join(mpi_home, 'include/mpicxx.h'), 'third_party/mpi/mpicxx.h')
1341 # TODO(gunan): avoid editing files in configure
1342 sed_in_place('third_party/mpi/mpi.bzl', 'MPI_LIB_IS_OPENMPI=True',
1343 'MPI_LIB_IS_OPENMPI=False')
1344
1345 if os.path.exists(os.path.join(mpi_home, 'lib/libmpi.so')):
1346 symlink_force(
1347 os.path.join(mpi_home, 'lib/libmpi.so'), 'third_party/mpi/libmpi.so')
1348 else:
1349 raise ValueError('Cannot find the MPI library file in %s/lib' % mpi_home)
1350
1351
A. Unique TensorFlower061c3592017-11-13 14:21:04 -08001352def set_grpc_build_flags():
1353 write_to_bazelrc('build --define grpc_no_ares=true')
1354
Michael Cased31531a2018-01-05 14:09:41 -08001355
Dandelion Man?90e42f32017-12-15 18:15:07 -08001356def set_windows_build_flags():
1357 if is_windows():
1358 # The non-monolithic build is not supported yet
1359 write_to_bazelrc('build --config monolithic')
1360 # Suppress warning messages
1361 write_to_bazelrc('build --copt=-w --host_copt=-w')
1362 # Output more verbose information when something goes wrong
1363 write_to_bazelrc('build --verbose_failures')
1364
A. Unique TensorFlower061c3592017-11-13 14:21:04 -08001365
Michael Cased31531a2018-01-05 14:09:41 -08001366def config_info_line(name, help_text):
1367 """Helper function to print formatted help text for Bazel config options."""
1368 print('\t--config=%-12s\t# %s' % (name, help_text))
1369
1370
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001371def main():
Shanqing Cai71445712018-03-12 19:33:52 -07001372 parser = argparse.ArgumentParser()
1373 parser.add_argument("--workspace",
1374 type=str,
1375 default=_TF_WORKSPACE_ROOT,
1376 help="The absolute path to your active Bazel workspace.")
1377 args = parser.parse_args()
1378
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001379 # Make a copy of os.environ to be clear when functions and getting and setting
1380 # environment variables.
1381 environ_cp = dict(os.environ)
1382
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001383 check_bazel_version('0.10.0')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001384
Shanqing Cai71445712018-03-12 19:33:52 -07001385 reset_tf_configure_bazelrc(args.workspace)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001386 cleanup_makefile()
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -07001387 setup_python(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001388
1389 if is_windows():
Benoit Steiner355e25e2017-10-24 19:47:46 -07001390 environ_cp['TF_NEED_S3'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001391 environ_cp['TF_NEED_GCP'] = '0'
1392 environ_cp['TF_NEED_HDFS'] = '0'
1393 environ_cp['TF_NEED_JEMALLOC'] = '0'
Michael Cased90054e2018-02-07 14:36:00 -08001394 environ_cp['TF_NEED_KAFKA'] = '0'
Yifei Fengb1d8c592017-11-22 13:42:21 -08001395 environ_cp['TF_NEED_OPENCL_SYCL'] = '0'
1396 environ_cp['TF_NEED_COMPUTECPP'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001397 environ_cp['TF_NEED_OPENCL'] = '0'
1398 environ_cp['TF_CUDA_CLANG'] = '0'
Guangda Lai76f69382018-01-25 23:59:19 -08001399 environ_cp['TF_NEED_TENSORRT'] = '0'
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001400 # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on
1401 # Windows.
1402 environ_cp['TF_DOWNLOAD_CLANG'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001403
1404 if is_macos():
1405 environ_cp['TF_NEED_JEMALLOC'] = '0'
Guangda Lai76f69382018-01-25 23:59:19 -08001406 environ_cp['TF_NEED_TENSORRT'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001407
1408 set_build_var(environ_cp, 'TF_NEED_JEMALLOC', 'jemalloc as malloc',
1409 'with_jemalloc', True)
1410 set_build_var(environ_cp, 'TF_NEED_GCP', 'Google Cloud Platform',
Benoit Steiner355e25e2017-10-24 19:47:46 -07001411 'with_gcp_support', True, 'gcp')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001412 set_build_var(environ_cp, 'TF_NEED_HDFS', 'Hadoop File System',
Benoit Steiner355e25e2017-10-24 19:47:46 -07001413 'with_hdfs_support', True, 'hdfs')
Michael Case7e4e3362017-10-09 13:31:15 -07001414 set_build_var(environ_cp, 'TF_NEED_S3', 'Amazon S3 File System',
1415 'with_s3_support', True, 's3')
Michael Cased90054e2018-02-07 14:36:00 -08001416 set_build_var(environ_cp, 'TF_NEED_KAFKA', 'Apache Kafka Platform',
1417 'with_kafka_support', False, 'kafka')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001418 set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
Michael Case98850a52017-09-14 13:35:57 -07001419 False, 'xla')
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -07001420 set_build_var(environ_cp, 'TF_NEED_GDR', 'GDR', 'with_gdr_support',
Michael Case98850a52017-09-14 13:35:57 -07001421 False, 'gdr')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001422 set_build_var(environ_cp, 'TF_NEED_VERBS', 'VERBS', 'with_verbs_support',
Michael Case98850a52017-09-14 13:35:57 -07001423 False, 'verbs')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001424
Yifei Fengb1d8c592017-11-22 13:42:21 -08001425 set_action_env_var(environ_cp, 'TF_NEED_OPENCL_SYCL', 'OpenCL SYCL', False)
1426 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1':
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001427 set_host_cxx_compiler(environ_cp)
1428 set_host_c_compiler(environ_cp)
Yifei Fengb1d8c592017-11-22 13:42:21 -08001429 set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True)
1430 if environ_cp.get('TF_NEED_COMPUTECPP') == '1':
1431 set_computecpp_toolkit_path(environ_cp)
1432 else:
1433 set_trisycl_include_dir(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001434
1435 set_action_env_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False)
A. Unique TensorFlower24cbb2a2017-09-08 07:45:44 -07001436 if (environ_cp.get('TF_NEED_CUDA') == '1' and
1437 'TF_CUDA_CONFIG_REPO' not in environ_cp):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001438 set_tf_cuda_version(environ_cp)
Yifei Fengb1d8c592017-11-22 13:42:21 -08001439 set_tf_cudnn_version(environ_cp)
Guangda Lai76f69382018-01-25 23:59:19 -08001440 if is_linux():
1441 set_tf_tensorrt_install_path(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001442 set_tf_cuda_compute_capabilities(environ_cp)
Ankur Taly0e6f39d2018-02-16 18:22:55 -08001443 if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get(
1444 'LD_LIBRARY_PATH') != '1':
1445 write_action_env_to_bazelrc('LD_LIBRARY_PATH',
1446 environ_cp.get('LD_LIBRARY_PATH'))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001447
1448 set_tf_cuda_clang(environ_cp)
1449 if environ_cp.get('TF_CUDA_CLANG') == '1':
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001450 # Ask whether we should download the clang toolchain.
1451 set_tf_download_clang(environ_cp)
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -08001452 if environ_cp.get('TF_DOWNLOAD_CLANG') != '1':
1453 # Set up which clang we should use as the cuda / host compiler.
1454 set_clang_cuda_compiler_path(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001455 else:
1456 # Set up which gcc nvcc should use as the host compiler
1457 # No need to set this on Windows
1458 if not is_windows():
1459 set_gcc_host_compiler_path(environ_cp)
1460 set_other_cuda_vars(environ_cp)
Ilya Biryukov9e651e42018-03-22 05:33:42 -07001461 else:
1462 # CUDA not required. Ask whether we should download the clang toolchain and
1463 # use it for the CPU build.
1464 set_tf_download_clang(environ_cp)
1465 if environ_cp.get('TF_DOWNLOAD_CLANG') == '1':
1466 write_to_bazelrc('build --config=download_clang')
1467 write_to_bazelrc('test --config=download_clang')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001468
1469 set_build_var(environ_cp, 'TF_NEED_MPI', 'MPI', 'with_mpi_support', False)
1470 if environ_cp.get('TF_NEED_MPI') == '1':
1471 set_mpi_home(environ_cp)
1472 set_other_mpi_vars(environ_cp)
1473
A. Unique TensorFlower061c3592017-11-13 14:21:04 -08001474 set_grpc_build_flags()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001475 set_cc_opt_flags(environ_cp)
Dandelion Man?90e42f32017-12-15 18:15:07 -08001476 set_windows_build_flags()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001477
Austin Anderson6afface2017-12-05 11:59:17 -08001478 if workspace_has_any_android_rule():
1479 print('The WORKSPACE file has at least one of ["android_sdk_repository", '
1480 '"android_ndk_repository"] already set. Will not ask to help '
1481 'configure the WORKSPACE. Please delete the existing rules to '
1482 'activate the helper.\n')
1483 else:
1484 if get_var(
1485 environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace',
1486 False,
1487 ('Would you like to interactively configure ./WORKSPACE for '
1488 'Android builds?'),
1489 'Searching for NDK and SDK installations.',
1490 'Not configuring the WORKSPACE for Android builds.'):
1491 create_android_ndk_rule(environ_cp)
1492 create_android_sdk_rule(environ_cp)
1493
Michael Cased31531a2018-01-05 14:09:41 -08001494 print('Preconfigured Bazel build configs. You can use any of the below by '
1495 'adding "--config=<>" to your build command. See tools/bazel.rc for '
1496 'more details.')
1497 config_info_line('mkl', 'Build with MKL support.')
1498 config_info_line('monolithic', 'Config for mostly static monolithic build.')
Austin Anderson6afface2017-12-05 11:59:17 -08001499
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001500if __name__ == '__main__':
1501 main()