blob: 073bccad2b106e0b6170e09f1184d1bf9490aadc [file] [log] [blame]
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""configure script to get build parameters from user."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
21import errno
22import os
23import platform
24import re
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070025import subprocess
26import sys
27
Andrew Sellec9885ea2017-11-06 09:37:03 -080028# pylint: disable=g-import-not-at-top
Jonathan Hseu008910f2017-08-25 14:01:05 -070029try:
30 from shutil import which
31except ImportError:
32 from distutils.spawn import find_executable as which
Andrew Sellec9885ea2017-11-06 09:37:03 -080033# pylint: enable=g-import-not-at-top
Jonathan Hseu008910f2017-08-25 14:01:05 -070034
Michael Casefe2c8d82017-10-02 13:54:34 -070035_TF_BAZELRC = os.path.join(os.path.dirname(os.path.abspath(__file__)),
36 '.tf_configure.bazelrc')
Austin Anderson6afface2017-12-05 11:59:17 -080037_TF_WORKSPACE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
38 'WORKSPACE')
Dandelion Man?90e42f32017-12-15 18:15:07 -080039_DEFAULT_CUDA_VERSION = '9.0'
40_DEFAULT_CUDNN_VERSION = '7'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070041_DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,5.2'
42_DEFAULT_CUDA_PATH = '/usr/local/cuda'
43_DEFAULT_CUDA_PATH_LINUX = '/opt/cuda'
44_DEFAULT_CUDA_PATH_WIN = ('C:/Program Files/NVIDIA GPU Computing '
45 'Toolkit/CUDA/v%s' % _DEFAULT_CUDA_VERSION)
46_TF_OPENCL_VERSION = '1.2'
47_DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp'
Yifei Fengb1d8c592017-11-22 13:42:21 -080048_DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include'
Austin Anderson6afface2017-12-05 11:59:17 -080049_SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15]
50
51_DEFAULT_PROMPT_ASK_ATTEMPTS = 10
52
53
54class UserInputError(Exception):
55 pass
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070056
57
58def is_windows():
59 return platform.system() == 'Windows'
60
61
62def is_linux():
63 return platform.system() == 'Linux'
64
65
66def is_macos():
67 return platform.system() == 'Darwin'
68
69
70def is_ppc64le():
71 return platform.machine() == 'ppc64le'
72
73
Jonathan Hseu008910f2017-08-25 14:01:05 -070074def is_cygwin():
75 return platform.system().startswith('CYGWIN_NT')
76
77
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -070078def get_input(question):
79 try:
80 try:
81 answer = raw_input(question)
82 except NameError:
83 answer = input(question) # pylint: disable=bad-builtin
84 except EOFError:
85 answer = ''
86 return answer
87
88
89def symlink_force(target, link_name):
90 """Force symlink, equivalent of 'ln -sf'.
91
92 Args:
93 target: items to link to.
94 link_name: name of the link.
95 """
96 try:
97 os.symlink(target, link_name)
98 except OSError as e:
99 if e.errno == errno.EEXIST:
100 os.remove(link_name)
101 os.symlink(target, link_name)
102 else:
103 raise e
104
105
106def sed_in_place(filename, old, new):
107 """Replace old string with new string in file.
108
109 Args:
110 filename: string for filename.
111 old: string to replace.
112 new: new string to replace to.
113 """
114 with open(filename, 'r') as f:
115 filedata = f.read()
116 newdata = filedata.replace(old, new)
117 with open(filename, 'w') as f:
118 f.write(newdata)
119
120
121def remove_line_with(filename, token):
122 """Remove lines that contain token from file.
123
124 Args:
125 filename: string for filename.
126 token: string token to check if to remove a line from file or not.
127 """
128 with open(filename, 'r') as f:
129 filedata = f.read()
130
131 with open(filename, 'w') as f:
132 for line in filedata.strip().split('\n'):
133 if token not in line:
134 f.write(line + '\n')
135
136
137def write_to_bazelrc(line):
138 with open(_TF_BAZELRC, 'a') as f:
139 f.write(line + '\n')
140
141
142def write_action_env_to_bazelrc(var_name, var):
143 write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var)))
144
145
Jonathan Hseu008910f2017-08-25 14:01:05 -0700146def run_shell(cmd, allow_non_zero=False):
147 if allow_non_zero:
148 try:
149 output = subprocess.check_output(cmd)
150 except subprocess.CalledProcessError as e:
151 output = e.output
152 else:
153 output = subprocess.check_output(cmd)
154 return output.decode('UTF-8').strip()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700155
156
157def cygpath(path):
158 """Convert path from posix to windows."""
Martin Wicked57572e2017-09-02 19:21:45 -0700159 return os.path.abspath(path).replace('\\', '/')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700160
161
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700162def get_python_path(environ_cp, python_bin_path):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700163 """Get the python site package paths."""
164 python_paths = []
165 if environ_cp.get('PYTHONPATH'):
166 python_paths = environ_cp.get('PYTHONPATH').split(':')
167 try:
Jonathan Hseu008910f2017-08-25 14:01:05 -0700168 library_paths = run_shell(
169 [python_bin_path, '-c',
Austin Anderson6afface2017-12-05 11:59:17 -0800170 'import site; print("\\n".join(site.getsitepackages()))']).split('\n')
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700171 except subprocess.CalledProcessError:
Jonathan Hseu008910f2017-08-25 14:01:05 -0700172 library_paths = [run_shell(
173 [python_bin_path, '-c',
174 'from distutils.sysconfig import get_python_lib;'
175 'print(get_python_lib())'])]
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700176
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700177 all_paths = set(python_paths + library_paths)
178
179 paths = []
180 for path in all_paths:
181 if os.path.isdir(path):
182 paths.append(path)
183 return paths
184
185
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700186def get_python_major_version(python_bin_path):
187 """Get the python major version."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700188 return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])'])
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700189
190
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -0700191def setup_python(environ_cp):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700192 """Setup python related env variables."""
193 # Get PYTHON_BIN_PATH, default is the current running python.
194 default_python_bin_path = sys.executable
195 ask_python_bin_path = ('Please specify the location of python. [Default is '
196 '%s]: ') % default_python_bin_path
197 while True:
198 python_bin_path = get_from_env_or_user_or_default(
199 environ_cp, 'PYTHON_BIN_PATH', ask_python_bin_path,
200 default_python_bin_path)
201 # Check if the path is valid
Jonathan Hseu008910f2017-08-25 14:01:05 -0700202 if os.path.isfile(python_bin_path) and os.access(
203 python_bin_path, os.X_OK):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700204 break
205 elif not os.path.exists(python_bin_path):
206 print('Invalid python path: %s cannot be found.' % python_bin_path)
207 else:
208 print('%s is not executable. Is it the python binary?' % python_bin_path)
209 environ_cp['PYTHON_BIN_PATH'] = ''
210
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700211 # Convert python path to Windows style before checking lib and version
Martin Wicked57572e2017-09-02 19:21:45 -0700212 if is_windows() or is_cygwin():
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700213 python_bin_path = cygpath(python_bin_path)
214
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700215 # Get PYTHON_LIB_PATH
216 python_lib_path = environ_cp.get('PYTHON_LIB_PATH')
217 if not python_lib_path:
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700218 python_lib_paths = get_python_path(environ_cp, python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700219 if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1':
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700220 python_lib_path = python_lib_paths[0]
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700221 else:
Jonathan Hseu008910f2017-08-25 14:01:05 -0700222 print('Found possible Python library paths:\n %s' %
223 '\n '.join(python_lib_paths))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700224 default_python_lib_path = python_lib_paths[0]
225 python_lib_path = get_input(
Jonathan Hseu008910f2017-08-25 14:01:05 -0700226 'Please input the desired Python library path to use. '
227 'Default is [%s]\n' % python_lib_paths[0])
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700228 if not python_lib_path:
229 python_lib_path = default_python_lib_path
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700230 environ_cp['PYTHON_LIB_PATH'] = python_lib_path
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700231
Andrew Harp6e3e7d12017-08-21 12:10:44 -0700232 python_major_version = get_python_major_version(python_bin_path)
233
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700234 # Convert python path to Windows style before writing into bazel.rc
Martin Wicked57572e2017-09-02 19:21:45 -0700235 if is_windows() or is_cygwin():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700236 python_lib_path = cygpath(python_lib_path)
237
238 # Set-up env variables used by python_configure.bzl
239 write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path)
240 write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700241 write_to_bazelrc('build --force_python=py%s' % python_major_version)
242 write_to_bazelrc('build --host_force_python=py%s' % python_major_version)
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -0700243 write_to_bazelrc('build --python_path=\"%s"' % python_bin_path)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700244 environ_cp['PYTHON_BIN_PATH'] = python_bin_path
245
246 # Write tools/python_bin_path.sh
247 with open('tools/python_bin_path.sh', 'w') as f:
248 f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path)
249
250
251def reset_tf_configure_bazelrc():
252 """Reset file that contains customized config settings."""
253 open(_TF_BAZELRC, 'w').close()
254
255 home = os.path.expanduser('~')
256 if not os.path.exists('.bazelrc'):
257 if os.path.exists(os.path.join(home, '.bazelrc')):
258 with open('.bazelrc', 'a') as f:
Shanqing Caie2e3a942017-09-25 19:35:53 -0700259 f.write('import %s/.bazelrc\n' % home.replace('\\', '/'))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700260 else:
261 open('.bazelrc', 'w').close()
262
263 remove_line_with('.bazelrc', 'tf_configure')
264 with open('.bazelrc', 'a') as f:
265 f.write('import %workspace%/.tf_configure.bazelrc\n')
266
267
268def run_gen_git_source(environ_cp):
269 """Run the gen_git_source to create links.
270
271 The links are for bazel to track dependencies for git hash propagation.
272
273 Args:
274 environ_cp: copy of the os.environ.
275 """
Vijay Vasudevana1fba7f2017-07-28 10:58:56 -0700276 cmd = '"%s" tensorflow/tools/git/gen_git_source.py --configure %s' % (
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700277 environ_cp.get('PYTHON_BIN_PATH'), os.getcwd())
278 os.system(cmd)
279
280
281def cleanup_makefile():
282 """Delete any leftover BUILD files from the Makefile build.
283
284 These files could interfere with Bazel parsing.
285 """
286 makefile_download_dir = 'tensorflow/contrib/makefile/downloads'
287 if os.path.isdir(makefile_download_dir):
288 for root, _, filenames in os.walk(makefile_download_dir):
289 for f in filenames:
290 if f.endswith('BUILD'):
291 os.remove(os.path.join(root, f))
292
293
294def get_var(environ_cp,
295 var_name,
296 query_item,
297 enabled_by_default,
298 question=None,
299 yes_reply=None,
300 no_reply=None):
301 """Get boolean input from user.
302
303 If var_name is not set in env, ask user to enable query_item or not. If the
304 response is empty, use the default.
305
306 Args:
307 environ_cp: copy of the os.environ.
308 var_name: string for name of environment variable, e.g. "TF_NEED_HDFS".
309 query_item: string for feature related to the variable, e.g. "Hadoop File
310 System".
311 enabled_by_default: boolean for default behavior.
312 question: optional string for how to ask for user input.
313 yes_reply: optionanl string for reply when feature is enabled.
314 no_reply: optional string for reply when feature is disabled.
315
316 Returns:
317 boolean value of the variable.
318 """
319 if not question:
320 question = 'Do you wish to build TensorFlow with %s support?' % query_item
321 if not yes_reply:
322 yes_reply = '%s support will be enabled for TensorFlow.' % query_item
323 if not no_reply:
324 no_reply = 'No %s' % yes_reply
325
326 yes_reply += '\n'
327 no_reply += '\n'
328
329 if enabled_by_default:
330 question += ' [Y/n]: '
331 else:
332 question += ' [y/N]: '
333
334 var = environ_cp.get(var_name)
335 while var is None:
336 user_input_origin = get_input(question)
337 user_input = user_input_origin.strip().lower()
338 if user_input == 'y':
339 print(yes_reply)
340 var = True
341 elif user_input == 'n':
342 print(no_reply)
343 var = False
344 elif not user_input:
345 if enabled_by_default:
346 print(yes_reply)
347 var = True
348 else:
349 print(no_reply)
350 var = False
351 else:
352 print('Invalid selection: %s' % user_input_origin)
353 return var
354
355
356def set_build_var(environ_cp, var_name, query_item, option_name,
Michael Case98850a52017-09-14 13:35:57 -0700357 enabled_by_default, bazel_config_name=None):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700358 """Set if query_item will be enabled for the build.
359
360 Ask user if query_item will be enabled. Default is used if no input is given.
361 Set subprocess environment variable and write to .bazelrc if enabled.
362
363 Args:
364 environ_cp: copy of the os.environ.
365 var_name: string for name of environment variable, e.g. "TF_NEED_HDFS".
366 query_item: string for feature related to the variable, e.g. "Hadoop File
367 System".
368 option_name: string for option to define in .bazelrc.
369 enabled_by_default: boolean for default behavior.
Michael Case98850a52017-09-14 13:35:57 -0700370 bazel_config_name: Name for Bazel --config argument to enable build feature.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700371 """
372
373 var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default)))
374 environ_cp[var_name] = var
375 if var == '1':
376 write_to_bazelrc('build --define %s=true' % option_name)
Michael Case98850a52017-09-14 13:35:57 -0700377 elif bazel_config_name is not None:
378 # TODO(mikecase): Migrate all users of configure.py to use --config Bazel
379 # options and not to set build configs through environment variables.
380 write_to_bazelrc('build:%s --define %s=true'
381 % (bazel_config_name, option_name))
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700382
383
384def set_action_env_var(environ_cp,
385 var_name,
386 query_item,
387 enabled_by_default,
388 question=None,
389 yes_reply=None,
390 no_reply=None):
391 """Set boolean action_env variable.
392
393 Ask user if query_item will be enabled. Default is used if no input is given.
394 Set environment variable and write to .bazelrc.
395
396 Args:
397 environ_cp: copy of the os.environ.
398 var_name: string for name of environment variable, e.g. "TF_NEED_HDFS".
399 query_item: string for feature related to the variable, e.g. "Hadoop File
400 System".
401 enabled_by_default: boolean for default behavior.
402 question: optional string for how to ask for user input.
403 yes_reply: optionanl string for reply when feature is enabled.
404 no_reply: optional string for reply when feature is disabled.
405 """
406 var = int(
407 get_var(environ_cp, var_name, query_item, enabled_by_default, question,
408 yes_reply, no_reply))
409
410 write_action_env_to_bazelrc(var_name, var)
411 environ_cp[var_name] = str(var)
412
413
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700414def convert_version_to_int(version):
415 """Convert a version number to a integer that can be used to compare.
416
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700417 Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The
418 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored.
419
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700420 Args:
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700421 version: a version to be converted
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700422
423 Returns:
424 An integer if converted successfully, otherwise return None.
425 """
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700426 version = version.split('-')[0]
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700427 version_segments = version.split('.')
428 for seg in version_segments:
429 if not seg.isdigit():
430 return None
431
432 version_str = ''.join(['%03d' % int(seg) for seg in version_segments])
433 return int(version_str)
434
435
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700436def check_bazel_version(min_version):
437 """Check installed bezel version is at least min_version.
438
439 Args:
440 min_version: string for minimum bazel version.
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700441
442 Returns:
443 The bazel version detected.
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700444 """
Jonathan Hseu008910f2017-08-25 14:01:05 -0700445 if which('bazel') is None:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700446 print('Cannot find bazel. Please install bazel.')
447 sys.exit(0)
Jonathan Hseu008910f2017-08-25 14:01:05 -0700448 curr_version = run_shell(['bazel', '--batch', 'version'])
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700449
450 for line in curr_version.split('\n'):
451 if 'Build label: ' in line:
452 curr_version = line.split('Build label: ')[1]
453 break
454
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700455 min_version_int = convert_version_to_int(min_version)
456 curr_version_int = convert_version_to_int(curr_version)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700457
458 # Check if current bazel version can be detected properly.
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700459 if not curr_version_int:
460 print('WARNING: current bazel installation is not a release version.')
461 print('Make sure you are running at least bazel %s' % min_version)
462 return curr_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700463
Michael Cased94271a2017-08-22 17:26:52 -0700464 print('You have bazel %s installed.' % curr_version)
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -0700465
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700466 if curr_version_int < min_version_int:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700467 print('Please upgrade your bazel installation to version %s or higher to '
468 'build TensorFlow!' % min_version)
469 sys.exit(0)
A. Unique TensorFlower6252d292017-08-04 00:52:34 -0700470 return curr_version
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700471
472
473def set_cc_opt_flags(environ_cp):
474 """Set up architecture-dependent optimization flags.
475
476 Also append CC optimization flags to bazel.rc..
477
478 Args:
479 environ_cp: copy of the os.environ.
480 """
481 if is_ppc64le():
482 # gcc on ppc64le does not support -march, use mcpu instead
483 default_cc_opt_flags = '-mcpu=native'
484 else:
485 default_cc_opt_flags = '-march=native'
486 question = ('Please specify optimization flags to use during compilation when'
487 ' bazel option "--config=opt" is specified [Default is %s]: '
488 ) % default_cc_opt_flags
489 cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS',
490 question, default_cc_opt_flags)
491 for opt in cc_opt_flags.split():
Michael Case00177422017-11-10 13:14:03 -0800492 write_to_bazelrc('build:opt --copt=%s' % opt)
493 # It should be safe on the same build host.
494 write_to_bazelrc('build:opt --host_copt=-march=native')
Michael Casebb3355d2017-11-09 08:46:31 -0800495 write_to_bazelrc('build:opt --define with_default_optimizations=true')
Michael Case00177422017-11-10 13:14:03 -0800496 # TODO(mikecase): Remove these default defines once we are able to get
497 # TF Lite targets building without them.
Andrew Selle0b154392017-11-10 10:35:35 -0800498 write_to_bazelrc('build --copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK')
499 write_to_bazelrc('build --host_copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700500
501
502def set_tf_cuda_clang(environ_cp):
503 """set TF_CUDA_CLANG action_env.
504
505 Args:
506 environ_cp: copy of the os.environ.
507 """
508 question = 'Do you want to use clang as CUDA compiler?'
509 yes_reply = 'Clang will be used as CUDA compiler.'
510 no_reply = 'nvcc will be used as CUDA compiler.'
511 set_action_env_var(
512 environ_cp,
513 'TF_CUDA_CLANG',
514 None,
515 False,
516 question=question,
517 yes_reply=yes_reply,
518 no_reply=no_reply)
519
520
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -0800521def set_tf_download_clang(environ_cp):
522 """Set TF_DOWNLOAD_CLANG action_env."""
523 question = 'Do you want to download a fresh release of clang? (Experimental)'
524 yes_reply = 'Clang will be downloaded and used to compile tensorflow.'
525 no_reply = 'Clang will not be downloaded.'
526 set_action_env_var(
527 environ_cp,
528 'TF_DOWNLOAD_CLANG',
529 None,
530 False,
531 question=question,
532 yes_reply=yes_reply,
533 no_reply=no_reply)
534
535
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700536def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var,
537 var_default):
538 """Get var_name either from env, or user or default.
539
540 If var_name has been set as environment variable, use the preset value, else
541 ask for user input. If no input is provided, the default is used.
542
543 Args:
544 environ_cp: copy of the os.environ.
545 var_name: string for name of environment variable, e.g. "TF_NEED_HDFS".
546 ask_for_var: string for how to ask for user input.
547 var_default: default value string.
548
549 Returns:
550 string value for var_name
551 """
552 var = environ_cp.get(var_name)
553 if not var:
554 var = get_input(ask_for_var)
Michael Cased94271a2017-08-22 17:26:52 -0700555 print('\n')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700556 if not var:
557 var = var_default
558 return var
559
560
561def set_clang_cuda_compiler_path(environ_cp):
562 """Set CLANG_CUDA_COMPILER_PATH."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700563 default_clang_path = which('clang') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700564 ask_clang_path = ('Please specify which clang should be used as device and '
565 'host compiler. [Default is %s]: ') % default_clang_path
566
567 while True:
568 clang_cuda_compiler_path = get_from_env_or_user_or_default(
569 environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path,
570 default_clang_path)
571 if os.path.exists(clang_cuda_compiler_path):
572 break
573
574 # Reset and retry
575 print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path)
576 environ_cp['CLANG_CUDA_COMPILER_PATH'] = ''
577
578 # Set CLANG_CUDA_COMPILER_PATH
579 environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path
580 write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH',
581 clang_cuda_compiler_path)
582
583
Austin Anderson6afface2017-12-05 11:59:17 -0800584def prompt_loop_or_load_from_env(
585 environ_cp,
586 var_name,
587 var_default,
588 ask_for_var,
589 check_success,
590 error_msg,
591 suppress_default_error=False,
592 n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS
593):
594 """Loop over user prompts for an ENV param until receiving a valid response.
595
596 For the env param var_name, read from the environment or verify user input
597 until receiving valid input. When done, set var_name in the environ_cp to its
598 new value.
599
600 Args:
601 environ_cp: (Dict) copy of the os.environ.
602 var_name: (String) string for name of environment variable, e.g. "TF_MYVAR".
603 var_default: (String) default value string.
604 ask_for_var: (String) string for how to ask for user input.
605 check_success: (Function) function that takes one argument and returns a
606 boolean. Should return True if the value provided is considered valid. May
607 contain a complex error message if error_msg does not provide enough
608 information. In that case, set suppress_default_error to True.
609 error_msg: (String) String with one and only one '%s'. Formatted with each
610 invalid response upon check_success(input) failure.
611 suppress_default_error: (Bool) Suppress the above error message in favor of
612 one from the check_success function.
613 n_ask_attempts: (Integer) Number of times to query for valid input before
614 raising an error and quitting.
615
616 Returns:
617 [String] The value of var_name after querying for input.
618
619 Raises:
620 UserInputError: if a query has been attempted n_ask_attempts times without
621 success, assume that the user has made a scripting error, and will continue
622 to provide invalid input. Raise the error to avoid infinitely looping.
623 """
624 default = environ_cp.get(var_name) or var_default
625 full_query = '%s [Default is %s]: ' % (
626 ask_for_var,
627 default,
628 )
629
630 for _ in range(n_ask_attempts):
631 val = get_from_env_or_user_or_default(environ_cp,
632 var_name,
633 full_query,
634 default)
635 if check_success(val):
636 break
637 if not suppress_default_error:
638 print(error_msg % val)
639 environ_cp[var_name] = ''
640 else:
641 raise UserInputError('Invalid %s setting was provided %d times in a row. '
642 'Assuming to be a scripting mistake.' %
643 (var_name, n_ask_attempts))
644
645 environ_cp[var_name] = val
646 return val
647
648
649def create_android_ndk_rule(environ_cp):
650 """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule."""
651 if is_windows() or is_cygwin():
652 default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' %
653 environ_cp['APPDATA'])
654 elif is_macos():
655 default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME']
656 else:
657 default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME']
658
659 def valid_ndk_path(path):
660 return (os.path.exists(path) and
661 os.path.exists(os.path.join(path, 'source.properties')))
662
663 android_ndk_home_path = prompt_loop_or_load_from_env(
664 environ_cp,
665 var_name='ANDROID_NDK_HOME',
666 var_default=default_ndk_path,
667 ask_for_var='Please specify the home path of the Android NDK to use.',
668 check_success=valid_ndk_path,
669 error_msg=('The path %s or its child file "source.properties" '
670 'does not exist.')
671 )
672
673 write_android_ndk_workspace_rule(android_ndk_home_path)
674
675
676def create_android_sdk_rule(environ_cp):
677 """Set Android variables and write Android SDK WORKSPACE rule."""
678 if is_windows() or is_cygwin():
679 default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA'])
680 elif is_macos():
681 default_sdk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME']
682 else:
683 default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME']
684
685 def valid_sdk_path(path):
686 return (os.path.exists(path) and
687 os.path.exists(os.path.join(path, 'platforms')) and
688 os.path.exists(os.path.join(path, 'build-tools')))
689
690 android_sdk_home_path = prompt_loop_or_load_from_env(
691 environ_cp,
692 var_name='ANDROID_SDK_HOME',
693 var_default=default_sdk_path,
694 ask_for_var='Please specify the home path of the Android SDK to use.',
695 check_success=valid_sdk_path,
696 error_msg=('Either %s does not exist, or it does not contain the '
697 'subdirectories "platforms" and "build-tools".'))
698
699 platforms = os.path.join(android_sdk_home_path, 'platforms')
700 api_levels = sorted(os.listdir(platforms))
701 api_levels = [x.replace('android-', '') for x in api_levels]
702
703 def valid_api_level(api_level):
704 return os.path.exists(os.path.join(android_sdk_home_path,
705 'platforms',
706 'android-' + api_level))
707
708 android_api_level = prompt_loop_or_load_from_env(
709 environ_cp,
710 var_name='ANDROID_API_LEVEL',
711 var_default=api_levels[-1],
712 ask_for_var=('Please specify the Android SDK API level to use. '
713 '[Available levels: %s]') % api_levels,
714 check_success=valid_api_level,
715 error_msg='Android-%s is not present in the SDK path.')
716
717 build_tools = os.path.join(android_sdk_home_path, 'build-tools')
718 versions = sorted(os.listdir(build_tools))
719
720 def valid_build_tools(version):
721 return os.path.exists(os.path.join(android_sdk_home_path,
722 'build-tools',
723 version))
724
725 android_build_tools_version = prompt_loop_or_load_from_env(
726 environ_cp,
727 var_name='ANDROID_BUILD_TOOLS_VERSION',
728 var_default=versions[-1],
729 ask_for_var=('Please specify an Android build tools version to use. '
730 '[Available versions: %s]') % versions,
731 check_success=valid_build_tools,
732 error_msg=('The selected SDK does not have build-tools version %s '
733 'available.'))
734
735 write_android_sdk_workspace_rule(android_sdk_home_path,
736 android_build_tools_version,
737 android_api_level)
738
739
740def write_android_sdk_workspace_rule(android_sdk_home_path,
741 android_build_tools_version,
742 android_api_level):
743 print('Writing android_sdk_workspace rule.\n')
744 with open(_TF_WORKSPACE, 'a') as f:
745 f.write("""
746android_sdk_repository(
747 name="androidsdk",
748 api_level=%s,
749 path="%s",
750 build_tools_version="%s")\n
751""" % (android_api_level, android_sdk_home_path, android_build_tools_version))
752
753
754def write_android_ndk_workspace_rule(android_ndk_home_path):
755 print('Writing android_ndk_workspace rule.')
756 ndk_api_level = check_ndk_level(android_ndk_home_path)
757 if int(ndk_api_level) not in _SUPPORTED_ANDROID_NDK_VERSIONS:
758 print('WARNING: The API level of the NDK in %s is %s, which is not '
759 'supported by Bazel (officially supported versions: %s). Please use '
760 'another version. Compiling Android targets may result in confusing '
761 'errors.\n' % (android_ndk_home_path, ndk_api_level,
762 _SUPPORTED_ANDROID_NDK_VERSIONS))
763 with open(_TF_WORKSPACE, 'a') as f:
764 f.write("""
765android_ndk_repository(
766 name="androidndk",
767 path="%s",
768 api_level=%s)\n
769""" % (android_ndk_home_path, ndk_api_level))
770
771
772def check_ndk_level(android_ndk_home_path):
773 """Check the revision number of an Android NDK path."""
774 properties_path = '%s/source.properties' % android_ndk_home_path
775 if is_windows() or is_cygwin():
776 properties_path = cygpath(properties_path)
777 with open(properties_path, 'r') as f:
778 filedata = f.read()
779
780 revision = re.search(r'Pkg.Revision = (\d+)', filedata)
781 if revision:
782 return revision.group(1)
783 return None
784
785
786def workspace_has_any_android_rule():
787 """Check the WORKSPACE for existing android_*_repository rules."""
788 with open(_TF_WORKSPACE, 'r') as f:
789 workspace = f.read()
790 has_any_rule = re.search(r'^android_[ns]dk_repository',
791 workspace,
792 re.MULTILINE)
793 return has_any_rule
794
795
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700796def set_gcc_host_compiler_path(environ_cp):
797 """Set GCC_HOST_COMPILER_PATH."""
Jonathan Hseu008910f2017-08-25 14:01:05 -0700798 default_gcc_host_compiler_path = which('gcc') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700799 cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH')
800
801 if os.path.islink(cuda_bin_symlink):
802 # os.readlink is only available in linux
Jonathan Hseu008910f2017-08-25 14:01:05 -0700803 default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700804
Austin Anderson6afface2017-12-05 11:59:17 -0800805 gcc_host_compiler_path = prompt_loop_or_load_from_env(
806 environ_cp,
807 var_name='GCC_HOST_COMPILER_PATH',
808 var_default=default_gcc_host_compiler_path,
809 ask_for_var=
810 'Please specify which gcc should be used by nvcc as the host compiler.',
811 check_success=os.path.exists,
812 error_msg='Invalid gcc path. %s cannot be found.',
813 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700814
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700815 write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path)
816
817
818def set_tf_cuda_version(environ_cp):
819 """Set CUDA_TOOLKIT_PATH and TF_CUDA_VERSION."""
820 ask_cuda_version = (
821 'Please specify the CUDA SDK version you want to use, '
822 'e.g. 7.0. [Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION
823
Austin Andersonf9a88f82017-12-13 11:49:40 -0800824 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700825 # Configure the Cuda SDK version to use.
826 tf_cuda_version = get_from_env_or_user_or_default(
827 environ_cp, 'TF_CUDA_VERSION', ask_cuda_version, _DEFAULT_CUDA_VERSION)
828
829 # Find out where the CUDA toolkit is installed
830 default_cuda_path = _DEFAULT_CUDA_PATH
Martin Wicked57572e2017-09-02 19:21:45 -0700831 if is_windows() or is_cygwin():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700832 default_cuda_path = cygpath(
833 environ_cp.get('CUDA_PATH', _DEFAULT_CUDA_PATH_WIN))
834 elif is_linux():
835 # If the default doesn't exist, try an alternative default.
836 if (not os.path.exists(default_cuda_path)
837 ) and os.path.exists(_DEFAULT_CUDA_PATH_LINUX):
838 default_cuda_path = _DEFAULT_CUDA_PATH_LINUX
839 ask_cuda_path = ('Please specify the location where CUDA %s toolkit is'
840 ' installed. Refer to README.md for more details. '
841 '[Default is %s]: ') % (tf_cuda_version, default_cuda_path)
842 cuda_toolkit_path = get_from_env_or_user_or_default(
843 environ_cp, 'CUDA_TOOLKIT_PATH', ask_cuda_path, default_cuda_path)
844
845 if is_windows():
846 cuda_rt_lib_path = 'lib/x64/cudart.lib'
847 elif is_linux():
848 cuda_rt_lib_path = 'lib64/libcudart.so.%s' % tf_cuda_version
849 elif is_macos():
850 cuda_rt_lib_path = 'lib/libcudart.%s.dylib' % tf_cuda_version
851
852 cuda_toolkit_path_full = os.path.join(cuda_toolkit_path, cuda_rt_lib_path)
853 if os.path.exists(cuda_toolkit_path_full):
854 break
855
856 # Reset and retry
857 print('Invalid path to CUDA %s toolkit. %s cannot be found' %
858 (tf_cuda_version, cuda_toolkit_path_full))
859 environ_cp['TF_CUDA_VERSION'] = ''
860 environ_cp['CUDA_TOOLKIT_PATH'] = ''
861
Austin Andersonf9a88f82017-12-13 11:49:40 -0800862 else:
863 raise UserInputError('Invalid TF_CUDA_SETTING setting was provided %d '
864 'times in a row. Assuming to be a scripting mistake.' %
865 _DEFAULT_PROMPT_ASK_ATTEMPTS)
866
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700867 # Set CUDA_TOOLKIT_PATH and TF_CUDA_VERSION
868 environ_cp['CUDA_TOOLKIT_PATH'] = cuda_toolkit_path
869 write_action_env_to_bazelrc('CUDA_TOOLKIT_PATH', cuda_toolkit_path)
870 environ_cp['TF_CUDA_VERSION'] = tf_cuda_version
871 write_action_env_to_bazelrc('TF_CUDA_VERSION', tf_cuda_version)
872
873
Yifei Fengb1d8c592017-11-22 13:42:21 -0800874def set_tf_cudnn_version(environ_cp):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700875 """Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION."""
876 ask_cudnn_version = (
Jonathan Hseu008910f2017-08-25 14:01:05 -0700877 'Please specify the cuDNN version you want to use. '
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700878 '[Leave empty to default to cuDNN %s.0]: ') % _DEFAULT_CUDNN_VERSION
879
Austin Andersonf9a88f82017-12-13 11:49:40 -0800880 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700881 tf_cudnn_version = get_from_env_or_user_or_default(
882 environ_cp, 'TF_CUDNN_VERSION', ask_cudnn_version,
883 _DEFAULT_CUDNN_VERSION)
884
885 default_cudnn_path = environ_cp.get('CUDA_TOOLKIT_PATH')
886 ask_cudnn_path = (r'Please specify the location where cuDNN %s library is '
887 'installed. Refer to README.md for more details. [Default'
888 ' is %s]:') % (tf_cudnn_version, default_cudnn_path)
889 cudnn_install_path = get_from_env_or_user_or_default(
890 environ_cp, 'CUDNN_INSTALL_PATH', ask_cudnn_path, default_cudnn_path)
891
892 # Result returned from "read" will be used unexpanded. That make "~"
893 # unusable. Going through one more level of expansion to handle that.
894 cudnn_install_path = os.path.realpath(
895 os.path.expanduser(cudnn_install_path))
Martin Wicked57572e2017-09-02 19:21:45 -0700896 if is_windows() or is_cygwin():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700897 cudnn_install_path = cygpath(cudnn_install_path)
898
899 if is_windows():
900 cuda_dnn_lib_path = 'lib/x64/cudnn.lib'
901 cuda_dnn_lib_alt_path = 'lib/x64/cudnn.lib'
902 elif is_linux():
903 cuda_dnn_lib_path = 'lib64/libcudnn.so.%s' % tf_cudnn_version
904 cuda_dnn_lib_alt_path = 'libcudnn.so.%s' % tf_cudnn_version
905 elif is_macos():
906 cuda_dnn_lib_path = 'lib/libcudnn.%s.dylib' % tf_cudnn_version
907 cuda_dnn_lib_alt_path = 'libcudnn.%s.dylib' % tf_cudnn_version
908
909 cuda_dnn_lib_path_full = os.path.join(cudnn_install_path, cuda_dnn_lib_path)
910 cuda_dnn_lib_alt_path_full = os.path.join(cudnn_install_path,
911 cuda_dnn_lib_alt_path)
912 if os.path.exists(cuda_dnn_lib_path_full) or os.path.exists(
913 cuda_dnn_lib_alt_path_full):
914 break
915
916 # Try another alternative for Linux
917 if is_linux():
Jonathan Hseu008910f2017-08-25 14:01:05 -0700918 ldconfig_bin = which('ldconfig') or '/sbin/ldconfig'
919 cudnn_path_from_ldconfig = run_shell([ldconfig_bin, '-p'])
920 cudnn_path_from_ldconfig = re.search('.*libcudnn.so .* => (.*)',
A. Unique TensorFlowere7223582017-09-06 17:57:04 -0700921 cudnn_path_from_ldconfig)
922 if cudnn_path_from_ldconfig:
923 cudnn_path_from_ldconfig = cudnn_path_from_ldconfig.group(1)
924 if os.path.exists('%s.%s' % (cudnn_path_from_ldconfig,
925 tf_cudnn_version)):
926 cudnn_install_path = os.path.dirname(cudnn_path_from_ldconfig)
927 break
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700928
929 # Reset and Retry
930 print(
931 'Invalid path to cuDNN %s toolkit. None of the following files can be '
932 'found:' % tf_cudnn_version)
933 print(cuda_dnn_lib_path_full)
934 print(cuda_dnn_lib_alt_path_full)
935 if is_linux():
936 print('%s.%s' % (cudnn_path_from_ldconfig, tf_cudnn_version))
937
938 environ_cp['TF_CUDNN_VERSION'] = ''
Austin Andersonf9a88f82017-12-13 11:49:40 -0800939 else:
940 raise UserInputError('Invalid TF_CUDNN setting was provided %d '
941 'times in a row. Assuming to be a scripting mistake.' %
942 _DEFAULT_PROMPT_ASK_ATTEMPTS)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700943
944 # Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION
945 environ_cp['CUDNN_INSTALL_PATH'] = cudnn_install_path
946 write_action_env_to_bazelrc('CUDNN_INSTALL_PATH', cudnn_install_path)
947 environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version
948 write_action_env_to_bazelrc('TF_CUDNN_VERSION', tf_cudnn_version)
949
950
951def get_native_cuda_compute_capabilities(environ_cp):
952 """Get native cuda compute capabilities.
953
954 Args:
955 environ_cp: copy of the os.environ.
956 Returns:
957 string of native cuda compute capabilities, separated by comma.
958 """
959 device_query_bin = os.path.join(
960 environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery')
Jonathan Hseu008910f2017-08-25 14:01:05 -0700961 if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK):
962 try:
963 output = run_shell(device_query_bin).split('\n')
964 pattern = re.compile('[0-9]*\\.[0-9]*')
965 output = [pattern.search(x) for x in output if 'Capability' in x]
966 output = ','.join(x.group() for x in output if x is not None)
967 except subprocess.CalledProcessError:
968 output = ''
969 else:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -0700970 output = ''
971 return output
972
973
974def set_tf_cuda_compute_capabilities(environ_cp):
975 """Set TF_CUDA_COMPUTE_CAPABILITIES."""
976 while True:
977 native_cuda_compute_capabilities = get_native_cuda_compute_capabilities(
978 environ_cp)
979 if not native_cuda_compute_capabilities:
980 default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES
981 else:
982 default_cuda_compute_capabilities = native_cuda_compute_capabilities
983
984 ask_cuda_compute_capabilities = (
985 'Please specify a list of comma-separated '
986 'Cuda compute capabilities you want to '
987 'build with.\nYou can find the compute '
988 'capability of your device at: '
989 'https://developer.nvidia.com/cuda-gpus.\nPlease'
990 ' note that each additional compute '
991 'capability significantly increases your '
992 'build time and binary size. [Default is: %s]' %
993 default_cuda_compute_capabilities)
994 tf_cuda_compute_capabilities = get_from_env_or_user_or_default(
995 environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES',
996 ask_cuda_compute_capabilities, default_cuda_compute_capabilities)
997 # Check whether all capabilities from the input is valid
998 all_valid = True
999 for compute_capability in tf_cuda_compute_capabilities.split(','):
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001000 m = re.match('[0-9]+.[0-9]+', compute_capability)
1001 if not m:
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001002 print('Invalid compute capability: ' % compute_capability)
1003 all_valid = False
Andrew Harp6e3e7d12017-08-21 12:10:44 -07001004 else:
1005 ver = int(m.group(0).split('.')[0])
1006 if ver < 3:
1007 print('Only compute capabilities 3.0 or higher are supported.')
1008 all_valid = False
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001009
1010 if all_valid:
1011 break
1012
1013 # Reset and Retry
1014 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = ''
1015
1016 # Set TF_CUDA_COMPUTE_CAPABILITIES
1017 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities
1018 write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES',
1019 tf_cuda_compute_capabilities)
1020
1021
1022def set_other_cuda_vars(environ_cp):
1023 """Set other CUDA related variables."""
1024 if is_windows():
1025 # The following three variables are needed for MSVC toolchain configuration
1026 # in Bazel
1027 environ_cp['CUDA_PATH'] = environ_cp.get('CUDA_TOOLKIT_PATH')
1028 environ_cp['CUDA_COMPUTE_CAPABILITIES'] = environ_cp.get(
1029 'TF_CUDA_COMPUTE_CAPABILITIES')
1030 environ_cp['NO_WHOLE_ARCHIVE_OPTION'] = 1
1031 write_action_env_to_bazelrc('CUDA_PATH', environ_cp.get('CUDA_PATH'))
1032 write_action_env_to_bazelrc('CUDA_COMPUTE_CAPABILITIE',
1033 environ_cp.get('CUDA_COMPUTE_CAPABILITIE'))
1034 write_action_env_to_bazelrc('NO_WHOLE_ARCHIVE_OPTION',
1035 environ_cp.get('NO_WHOLE_ARCHIVE_OPTION'))
1036 write_to_bazelrc('build --config=win-cuda')
1037 write_to_bazelrc('test --config=win-cuda')
1038 else:
1039 # If CUDA is enabled, always use GPU during build and test.
1040 if environ_cp.get('TF_CUDA_CLANG') == '1':
1041 write_to_bazelrc('build --config=cuda_clang')
1042 write_to_bazelrc('test --config=cuda_clang')
1043 else:
1044 write_to_bazelrc('build --config=cuda')
1045 write_to_bazelrc('test --config=cuda')
1046
1047
1048def set_host_cxx_compiler(environ_cp):
1049 """Set HOST_CXX_COMPILER."""
Jonathan Hseu008910f2017-08-25 14:01:05 -07001050 default_cxx_host_compiler = which('g++') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001051
Austin Anderson6afface2017-12-05 11:59:17 -08001052 host_cxx_compiler = prompt_loop_or_load_from_env(
1053 environ_cp,
1054 var_name='HOST_CXX_COMPILER',
1055 var_default=default_cxx_host_compiler,
1056 ask_for_var=('Please specify which C++ compiler should be used as the '
1057 'host C++ compiler.'),
1058 check_success=os.path.exists,
1059 error_msg='Invalid C++ compiler path. %s cannot be found.',
1060 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001061
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001062 write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler)
1063
1064
1065def set_host_c_compiler(environ_cp):
1066 """Set HOST_C_COMPILER."""
Jonathan Hseu008910f2017-08-25 14:01:05 -07001067 default_c_host_compiler = which('gcc') or ''
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001068
Austin Anderson6afface2017-12-05 11:59:17 -08001069 host_c_compiler = prompt_loop_or_load_from_env(
1070 environ_cp,
1071 var_name='HOST_C_COMPILER',
1072 var_default=default_c_host_compiler,
1073 ask_for_var=('Please specify which C compiler should be used as the host'
1074 'C compiler.'),
1075 check_success=os.path.exists,
1076 error_msg='Invalid C compiler path. %s cannot be found.',
1077 )
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001078
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001079 write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler)
1080
1081
1082def set_computecpp_toolkit_path(environ_cp):
1083 """Set COMPUTECPP_TOOLKIT_PATH."""
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001084
Austin Anderson6afface2017-12-05 11:59:17 -08001085 def toolkit_exists(toolkit_path):
1086 """Check if a computecpp toolkit path is valid."""
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001087 if is_linux():
1088 sycl_rt_lib_path = 'lib/libComputeCpp.so'
1089 else:
1090 sycl_rt_lib_path = ''
1091
Austin Anderson6afface2017-12-05 11:59:17 -08001092 sycl_rt_lib_path_full = os.path.join(toolkit_path,
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001093 sycl_rt_lib_path)
Austin Anderson6afface2017-12-05 11:59:17 -08001094 exists = os.path.exists(sycl_rt_lib_path_full)
1095 if not exists:
1096 print('Invalid SYCL %s library path. %s cannot be found' %
1097 (_TF_OPENCL_VERSION, sycl_rt_lib_path_full))
1098 return exists
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001099
Austin Anderson6afface2017-12-05 11:59:17 -08001100 computecpp_toolkit_path = prompt_loop_or_load_from_env(
1101 environ_cp,
1102 var_name='COMPUTECPP_TOOLKIT_PATH',
1103 var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH,
1104 ask_for_var=(
1105 'Please specify the location where ComputeCpp for SYCL %s is '
1106 'installed.' % _TF_OPENCL_VERSION),
1107 check_success=toolkit_exists,
1108 error_msg='Invalid SYCL compiler path. %s cannot be found.',
1109 suppress_default_error=True)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001110
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001111 write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH',
1112 computecpp_toolkit_path)
1113
Dandelion Man?90e42f32017-12-15 18:15:07 -08001114def set_trisycl_include_dir(environ_cp):
1115 """Set TRISYCL_INCLUDE_DIR"""
1116 ask_trisycl_include_dir = ('Please specify the location of the triSYCL '
1117 'include directory. (Use --config=sycl_trisycl '
1118 'when building with Bazel) '
1119 '[Default is %s]: '
1120 ) % (_DEFAULT_TRISYCL_INCLUDE_DIR)
1121 while True:
1122 trisycl_include_dir = get_from_env_or_user_or_default(
1123 environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir,
1124 _DEFAULT_TRISYCL_INCLUDE_DIR)
1125 if os.path.exists(trisycl_include_dir):
1126 break
1127
1128 print('Invalid triSYCL include directory, %s cannot be found'
1129 % (trisycl_include_dir))
1130
1131 # Set TRISYCL_INCLUDE_DIR
1132 environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir
1133 write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR',
1134 trisycl_include_dir)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001135
Yifei Fengb1d8c592017-11-22 13:42:21 -08001136def set_trisycl_include_dir(environ_cp):
1137 """Set TRISYCL_INCLUDE_DIR."""
Yifei Fengb1d8c592017-11-22 13:42:21 -08001138
Austin Anderson6afface2017-12-05 11:59:17 -08001139 trisycl_include_dir = prompt_loop_or_load_from_env(
1140 environ_cp,
1141 var_name='TRISYCL_INCLUDE_DIR',
1142 var_default=_DEFAULT_TRISYCL_INCLUDE_DIR,
1143 ask_for_var=('Please specify the location of the triSYCL include '
1144 'directory. (Use --config=sycl_trisycl when building with '
1145 'Bazel)'),
1146 check_success=os.path.exists,
1147 error_msg='Invalid trySYCL include directory. %s cannot be found.',
1148 suppress_default_error=True)
Yifei Fengb1d8c592017-11-22 13:42:21 -08001149
Yifei Fengb1d8c592017-11-22 13:42:21 -08001150 write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', trisycl_include_dir)
1151
1152
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001153def set_mpi_home(environ_cp):
1154 """Set MPI_HOME."""
Austin Anderson6afface2017-12-05 11:59:17 -08001155
Jonathan Hseu008910f2017-08-25 14:01:05 -07001156 default_mpi_home = which('mpirun') or which('mpiexec') or ''
1157 default_mpi_home = os.path.dirname(os.path.dirname(default_mpi_home))
1158
Austin Anderson6afface2017-12-05 11:59:17 -08001159 def valid_mpi_path(mpi_home):
1160 exists = (os.path.exists(os.path.join(mpi_home, 'include')) and
1161 os.path.exists(os.path.join(mpi_home, 'lib')))
1162 if not exists:
1163 print('Invalid path to the MPI Toolkit. %s or %s cannot be found' %
1164 (os.path.join(mpi_home, 'include'),
1165 os.path.exists(os.path.join(mpi_home, 'lib'))))
1166 return exists
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001167
Austin Anderson6afface2017-12-05 11:59:17 -08001168 _ = prompt_loop_or_load_from_env(
1169 environ_cp,
1170 var_name='MPI_HOME',
1171 var_default=default_mpi_home,
1172 ask_for_var='Please specify the MPI toolkit folder.',
1173 check_success=valid_mpi_path,
1174 error_msg='',
1175 suppress_default_error=True)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001176
1177
1178def set_other_mpi_vars(environ_cp):
1179 """Set other MPI related variables."""
1180 # Link the MPI header files
1181 mpi_home = environ_cp.get('MPI_HOME')
1182 symlink_force('%s/include/mpi.h' % mpi_home, 'third_party/mpi/mpi.h')
1183
1184 # Determine if we use OpenMPI or MVAPICH, these require different header files
1185 # to be included here to make bazel dependency checker happy
1186 if os.path.exists(os.path.join(mpi_home, 'include/mpi_portable_platform.h')):
1187 symlink_force(
1188 os.path.join(mpi_home, 'include/mpi_portable_platform.h'),
1189 'third_party/mpi/mpi_portable_platform.h')
1190 # TODO(gunan): avoid editing files in configure
1191 sed_in_place('third_party/mpi/mpi.bzl', 'MPI_LIB_IS_OPENMPI=False',
1192 'MPI_LIB_IS_OPENMPI=True')
1193 else:
1194 # MVAPICH / MPICH
1195 symlink_force(
1196 os.path.join(mpi_home, 'include/mpio.h'), 'third_party/mpi/mpio.h')
1197 symlink_force(
1198 os.path.join(mpi_home, 'include/mpicxx.h'), 'third_party/mpi/mpicxx.h')
1199 # TODO(gunan): avoid editing files in configure
1200 sed_in_place('third_party/mpi/mpi.bzl', 'MPI_LIB_IS_OPENMPI=True',
1201 'MPI_LIB_IS_OPENMPI=False')
1202
1203 if os.path.exists(os.path.join(mpi_home, 'lib/libmpi.so')):
1204 symlink_force(
1205 os.path.join(mpi_home, 'lib/libmpi.so'), 'third_party/mpi/libmpi.so')
1206 else:
1207 raise ValueError('Cannot find the MPI library file in %s/lib' % mpi_home)
1208
1209
1210def set_mkl():
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001211 write_to_bazelrc('build:mkl --define using_mkl=true')
1212 write_to_bazelrc('build:mkl -c opt')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001213 print(
1214 'Add "--config=mkl" to your bazel command to build with MKL '
1215 'support.\nPlease note that MKL on MacOS or windows is still not '
1216 'supported.\nIf you would like to use a local MKL instead of '
1217 'downloading, please set the environment variable \"TF_MKL_ROOT\" every '
Austin Anderson6afface2017-12-05 11:59:17 -08001218 'time before build.\n')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001219
1220
Allen Lavoie5c7f9e32017-09-21 11:29:45 -07001221def set_monolithic():
1222 # Add --config=monolithic to your bazel command to use a mostly-static
1223 # build and disable modular op registration support (this will revert to
1224 # loading TensorFlow with RTLD_GLOBAL in Python). By default (without
1225 # --config=monolithic), TensorFlow will build with a dependence on
1226 # //tensorflow:libtensorflow_framework.so.
1227 write_to_bazelrc('build:monolithic --define framework_shared_object=false')
1228 # For projects which use TensorFlow as part of a Bazel build process, putting
1229 # nothing in a bazelrc will default to a monolithic build. The following line
1230 # opts in to modular op registration support by default:
1231 write_to_bazelrc('build --define framework_shared_object=true')
1232
1233
Michael Casef1ecdd62017-10-24 18:07:59 -07001234def create_android_bazelrc_configs():
1235 # Flags for --config=android
1236 write_to_bazelrc('build:android --crosstool_top=//external:android/crosstool')
1237 write_to_bazelrc(
1238 'build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain')
1239 # Flags for --config=android_arm
1240 write_to_bazelrc('build:android_arm --config=android')
1241 write_to_bazelrc('build:android_arm --cpu=armeabi-v7a')
1242 # Flags for --config=android_arm64
1243 write_to_bazelrc('build:android_arm64 --config=android')
1244 write_to_bazelrc('build:android_arm64 --cpu=arm64-v8a')
1245
1246
A. Unique TensorFlower061c3592017-11-13 14:21:04 -08001247def set_grpc_build_flags():
1248 write_to_bazelrc('build --define grpc_no_ares=true')
1249
Dandelion Man?90e42f32017-12-15 18:15:07 -08001250def set_windows_build_flags():
1251 if is_windows():
1252 # The non-monolithic build is not supported yet
1253 write_to_bazelrc('build --config monolithic')
1254 # Suppress warning messages
1255 write_to_bazelrc('build --copt=-w --host_copt=-w')
1256 # Output more verbose information when something goes wrong
1257 write_to_bazelrc('build --verbose_failures')
1258
A. Unique TensorFlower061c3592017-11-13 14:21:04 -08001259
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001260def main():
1261 # Make a copy of os.environ to be clear when functions and getting and setting
1262 # environment variables.
1263 environ_cp = dict(os.environ)
1264
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -07001265 check_bazel_version('0.5.4')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001266
1267 reset_tf_configure_bazelrc()
1268 cleanup_makefile()
Gunhan Gulsoyed89a2b2017-09-19 18:36:26 -07001269 setup_python(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001270 run_gen_git_source(environ_cp)
1271
1272 if is_windows():
Benoit Steiner355e25e2017-10-24 19:47:46 -07001273 environ_cp['TF_NEED_S3'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001274 environ_cp['TF_NEED_GCP'] = '0'
1275 environ_cp['TF_NEED_HDFS'] = '0'
1276 environ_cp['TF_NEED_JEMALLOC'] = '0'
Yifei Fengb1d8c592017-11-22 13:42:21 -08001277 environ_cp['TF_NEED_OPENCL_SYCL'] = '0'
1278 environ_cp['TF_NEED_COMPUTECPP'] = '0'
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001279 environ_cp['TF_NEED_OPENCL'] = '0'
1280 environ_cp['TF_CUDA_CLANG'] = '0'
1281
1282 if is_macos():
1283 environ_cp['TF_NEED_JEMALLOC'] = '0'
1284
1285 set_build_var(environ_cp, 'TF_NEED_JEMALLOC', 'jemalloc as malloc',
1286 'with_jemalloc', True)
1287 set_build_var(environ_cp, 'TF_NEED_GCP', 'Google Cloud Platform',
Benoit Steiner355e25e2017-10-24 19:47:46 -07001288 'with_gcp_support', True, 'gcp')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001289 set_build_var(environ_cp, 'TF_NEED_HDFS', 'Hadoop File System',
Benoit Steiner355e25e2017-10-24 19:47:46 -07001290 'with_hdfs_support', True, 'hdfs')
Michael Case7e4e3362017-10-09 13:31:15 -07001291 set_build_var(environ_cp, 'TF_NEED_S3', 'Amazon S3 File System',
1292 'with_s3_support', True, 's3')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001293 set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
Michael Case98850a52017-09-14 13:35:57 -07001294 False, 'xla')
A. Unique TensorFlower28ce1d12017-08-15 12:08:29 -07001295 set_build_var(environ_cp, 'TF_NEED_GDR', 'GDR', 'with_gdr_support',
Michael Case98850a52017-09-14 13:35:57 -07001296 False, 'gdr')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001297 set_build_var(environ_cp, 'TF_NEED_VERBS', 'VERBS', 'with_verbs_support',
Michael Case98850a52017-09-14 13:35:57 -07001298 False, 'verbs')
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001299
Yifei Fengb1d8c592017-11-22 13:42:21 -08001300 set_action_env_var(environ_cp, 'TF_NEED_OPENCL_SYCL', 'OpenCL SYCL', False)
1301 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1':
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001302 set_host_cxx_compiler(environ_cp)
1303 set_host_c_compiler(environ_cp)
Yifei Fengb1d8c592017-11-22 13:42:21 -08001304 set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True)
1305 if environ_cp.get('TF_NEED_COMPUTECPP') == '1':
1306 set_computecpp_toolkit_path(environ_cp)
1307 else:
1308 set_trisycl_include_dir(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001309
1310 set_action_env_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False)
A. Unique TensorFlower24cbb2a2017-09-08 07:45:44 -07001311 if (environ_cp.get('TF_NEED_CUDA') == '1' and
1312 'TF_CUDA_CONFIG_REPO' not in environ_cp):
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001313 set_tf_cuda_version(environ_cp)
Yifei Fengb1d8c592017-11-22 13:42:21 -08001314 set_tf_cudnn_version(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001315 set_tf_cuda_compute_capabilities(environ_cp)
1316
1317 set_tf_cuda_clang(environ_cp)
1318 if environ_cp.get('TF_CUDA_CLANG') == '1':
A. Unique TensorFlowerfd29e952017-12-22 03:07:51 -08001319 if not is_windows():
1320 # Ask if we want to download clang release while building.
1321 set_tf_download_clang(environ_cp)
1322 else:
1323 # We use bazel's generated crosstool on Windows and there is no
1324 # way to provide downloaded toolchain for that yet.
1325 # TODO(ibiryukov): Investigate using clang as a cuda compiler on
1326 # Windows.
1327 environ_cp['TF_DOWNLOAD_CLANG'] = '0'
1328
1329 if environ_cp.get('TF_DOWNLOAD_CLANG') != '1':
1330 # Set up which clang we should use as the cuda / host compiler.
1331 set_clang_cuda_compiler_path(environ_cp)
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001332 else:
1333 # Set up which gcc nvcc should use as the host compiler
1334 # No need to set this on Windows
1335 if not is_windows():
1336 set_gcc_host_compiler_path(environ_cp)
1337 set_other_cuda_vars(environ_cp)
1338
1339 set_build_var(environ_cp, 'TF_NEED_MPI', 'MPI', 'with_mpi_support', False)
1340 if environ_cp.get('TF_NEED_MPI') == '1':
1341 set_mpi_home(environ_cp)
1342 set_other_mpi_vars(environ_cp)
1343
A. Unique TensorFlower061c3592017-11-13 14:21:04 -08001344 set_grpc_build_flags()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001345 set_cc_opt_flags(environ_cp)
1346 set_mkl()
Allen Lavoie5c7f9e32017-09-21 11:29:45 -07001347 set_monolithic()
Dandelion Man?90e42f32017-12-15 18:15:07 -08001348 set_windows_build_flags()
Michael Casef1ecdd62017-10-24 18:07:59 -07001349 create_android_bazelrc_configs()
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001350
Austin Anderson6afface2017-12-05 11:59:17 -08001351 if workspace_has_any_android_rule():
1352 print('The WORKSPACE file has at least one of ["android_sdk_repository", '
1353 '"android_ndk_repository"] already set. Will not ask to help '
1354 'configure the WORKSPACE. Please delete the existing rules to '
1355 'activate the helper.\n')
1356 else:
1357 if get_var(
1358 environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace',
1359 False,
1360 ('Would you like to interactively configure ./WORKSPACE for '
1361 'Android builds?'),
1362 'Searching for NDK and SDK installations.',
1363 'Not configuring the WORKSPACE for Android builds.'):
1364 create_android_ndk_rule(environ_cp)
1365 create_android_sdk_rule(environ_cp)
1366
1367
A. Unique TensorFlower73ea2872017-07-25 13:30:03 -07001368if __name__ == '__main__':
1369 main()