blob: 6ff206450273e8dc15462aba7842b9862c20832d [file] [log] [blame]
Han Shen91445322013-03-20 13:43:31 -07001#!/usr/bin/python
Han Shen819f8622014-04-08 16:57:38 -07002"""Script to bootstrap the chroot using new toolchain.
3
4This script allows you to build/install a customized version of gcc/binutils,
5either by specifying branch or a local directory.
6
7This script must be executed outside chroot.
8
9Below is some typical usage -
10
11## Build gcc located at /local/gcc/dir and do a bootstrap using the new compiler
12## for the chromeos root. The script tries to find a valid chromeos tree all
13## the way up from your current working directory.
14./build_tool.py --gcc_dir=/loca/gcc/dir --bootstrap
15
16## Build binutils, using remote branch "mobile_toolchain_v17" and do a bootstrap
17## using the new binutils for the chromeos root. The script tries to find a
18## valid chromeos tree all the way up from your current working directory.
19./build_tool.py --binutils_branch=cros/mobile_toolchain_v17 \
20 --chromeos_root=/chromeos/dir --bootstrap
21
22## Same as above except only do it for board daisy - no bootstrapping involved.
23./build_tool.py --binutils_branch=cros/mobile_toolchain_v16 \
24 --chromeos_root=/chromeos/dir --board=daisy
25"""
Han Shen91445322013-03-20 13:43:31 -070026
27__author__ = 'shenhan@google.com (Han Shen)'
28
29import optparse
30import os
31import re
Han Shen91445322013-03-20 13:43:31 -070032import sys
33
Han Shen819f8622014-04-08 16:57:38 -070034import repo_to_repo
Han Shen91445322013-03-20 13:43:31 -070035from utils import command_executer
36from utils import logger
37from utils import misc
38
Han Shen819f8622014-04-08 16:57:38 -070039REPO_PATH_PATTERN = 'src/third_party/{0}'
40TEMP_BRANCH_NAME = 'internal_testing_branch_no_use'
41CHROMIUMOS_OVERLAY_PATH = 'src/third_party/chromiumos-overlay'
42EBUILD_PATH_PATTERN = 'src/third_party/chromiumos-overlay/sys-devel/{0}'
43
Han Shen91445322013-03-20 13:43:31 -070044
45class Bootstrapper(object):
Han Shen819f8622014-04-08 16:57:38 -070046 """Class that handles bootstrap process.
47 """
48
Luis Lozanof2a3ef42015-12-15 13:49:30 -080049 def __init__(self,
50 chromeos_root,
51 gcc_branch=None,
52 gcc_dir=None,
53 binutils_branch=None,
54 binutils_dir=None,
55 board=None,
56 disable_2nd_bootstrap=False,
Han Shen03d30982014-06-12 11:22:29 -070057 setup_tool_ebuild_file_only=False):
Han Shen91445322013-03-20 13:43:31 -070058 self._chromeos_root = chromeos_root
Han Shen819f8622014-04-08 16:57:38 -070059
60 self._gcc_branch = gcc_branch
61 self._gcc_branch_tree = None
Han Shen91445322013-03-20 13:43:31 -070062 self._gcc_dir = gcc_dir
Han Shen91445322013-03-20 13:43:31 -070063 self._gcc_ebuild_file = None
64 self._gcc_ebuild_file_name = None
Han Shen819f8622014-04-08 16:57:38 -070065
66 self._binutils_branch = binutils_branch
67 self._binutils_branch_tree = None
68 self._binutils_dir = binutils_dir
69 self._binutils_ebuild_file = None
70 self._binutils_ebuild_file_name = None
71
72 self._setup_tool_ebuild_file_only = setup_tool_ebuild_file_only
73
74 self._ce = command_executer.GetCommandExecuter()
75 self._logger = logger.GetLogger()
76 self._board = board
Han Shen03d30982014-06-12 11:22:29 -070077 self._disable_2nd_bootstrap = disable_2nd_bootstrap
Han Shen819f8622014-04-08 16:57:38 -070078
79 def IsTreeSame(self, t1, t2):
80 diff = 'diff -qr -x .git -x .svn "{0}" "{1}"'.format(t1, t2)
81 if self._ce.RunCommand(diff, print_to_console=False) == 0:
82 self._logger.LogOutput('"{0}" and "{1}" are the same."'.format(t1, t2))
83 return True
84 self._logger.LogWarning('"{0}" and "{1}" are different."'.format(t1, t2))
85 return False
Han Shen91445322013-03-20 13:43:31 -070086
87 def SubmitToLocalBranch(self):
Han Shen819f8622014-04-08 16:57:38 -070088 """Copy source code to the chromium source tree and submit it locally."""
89 if self._gcc_dir:
Luis Lozanof2a3ef42015-12-15 13:49:30 -080090 if not self.SubmitToolToLocalBranch(tool_name='gcc',
91 tool_dir=self._gcc_dir):
Han Shen819f8622014-04-08 16:57:38 -070092 return False
93 self._gcc_branch = TEMP_BRANCH_NAME
Han Shen91445322013-03-20 13:43:31 -070094
Han Shen819f8622014-04-08 16:57:38 -070095 if self._binutils_dir:
Luis Lozanof2a3ef42015-12-15 13:49:30 -080096 if not self.SubmitToolToLocalBranch(tool_name='binutils',
97 tool_dir=self._binutils_dir):
Han Shen819f8622014-04-08 16:57:38 -070098 return False
99 self._binutils_branch = TEMP_BRANCH_NAME
100
101 return True
102
103 def SubmitToolToLocalBranch(self, tool_name, tool_dir):
104 """Copy the source code to local chromium source tree.
105
106 Args:
107 tool_name: either 'gcc' or 'binutils'
108 tool_dir: the tool source dir to be used
109 Returns:
110 True if all succeeded False otherwise.
111 """
112
113 # The next few steps creates an internal branch to sync with the tool dir
Han Shen91445322013-03-20 13:43:31 -0700114 # user provided.
Han Shen819f8622014-04-08 16:57:38 -0700115 chrome_tool_dir = self.GetChromeOsToolDir(tool_name)
Han Shen91445322013-03-20 13:43:31 -0700116
117 # 0. Test to see if git tree is free of local changes.
Han Shen819f8622014-04-08 16:57:38 -0700118 if not misc.IsGitTreeClean(chrome_tool_dir):
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800119 self._logger.LogError('Git repository "{0}" not clean, aborted.'.format(
120 chrome_tool_dir))
Han Shen91445322013-03-20 13:43:31 -0700121 return False
122
123 # 1. Checkout/create a (new) branch for testing.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800124 command = 'cd "{0}" && git checkout -B {1}'.format(chrome_tool_dir,
125 TEMP_BRANCH_NAME)
Han Shen91445322013-03-20 13:43:31 -0700126 ret = self._ce.RunCommand(command)
127 if ret:
128 self._logger.LogError('Failed to create a temp branch for test, aborted.')
129 return False
130
Han Shen819f8622014-04-08 16:57:38 -0700131 if self.IsTreeSame(tool_dir, chrome_tool_dir):
132 self._logger.LogOutput(
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800133 '"{0}" and "{1}" are the same, sync skipped.'.format(tool_dir,
134 chrome_tool_dir))
Han Shen819f8622014-04-08 16:57:38 -0700135 return True
Han Shen91445322013-03-20 13:43:31 -0700136
Han Shen819f8622014-04-08 16:57:38 -0700137 # 2. Sync sources from user provided tool dir to chromiumos tool git.
138 local_tool_repo = repo_to_repo.FileRepo(tool_dir)
139 chrome_tool_repo = repo_to_repo.GitRepo(chrome_tool_dir, TEMP_BRANCH_NAME)
140 chrome_tool_repo._root_dir = chrome_tool_dir
Han Shen03d30982014-06-12 11:22:29 -0700141 # Delete all stuff except '.git' before start mapping.
142 self._ce.RunCommand(
143 'cd {0} && find . -maxdepth 1 -not -name ".git" -not -name "." '
144 r'\( -type f -exec rm {{}} \; -o '
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800145 r' -type d -exec rm -fr {{}} \; \)'.format(chrome_tool_dir))
Han Shen819f8622014-04-08 16:57:38 -0700146 local_tool_repo.MapSources(chrome_tool_repo.GetRoot())
147
148 # 3. Ensure after sync tree is the same.
149 if self.IsTreeSame(tool_dir, chrome_tool_dir):
Han Shen91445322013-03-20 13:43:31 -0700150 self._logger.LogOutput('Sync successfully done.')
Han Shen819f8622014-04-08 16:57:38 -0700151 else:
152 self._logger.LogError('Sync not successful, aborted.')
153 return False
Han Shen91445322013-03-20 13:43:31 -0700154
155 # 4. Commit all changes.
Han Shen03d30982014-06-12 11:22:29 -0700156 # 4.1 Try to get some information about the tool dir we are using.
157 cmd = 'cd {0} && git log -1 --pretty=oneline'.format(tool_dir)
158 tool_dir_extra_info = None
Luis Lozano036c9232015-12-10 10:47:01 -0800159 ret, tool_dir_extra_info, _ = self._ce.RunCommandWOutput(
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800160 cmd,
161 print_to_console=False)
Han Shen03d30982014-06-12 11:22:29 -0700162 commit_message = 'Synced with tool source tree at - "{0}".'.format(tool_dir)
163 if not ret:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800164 commit_message += '\nGit log for {0}:\n{1}'.format(tool_dir,
165 tool_dir_extra_info)
Han Shen03d30982014-06-12 11:22:29 -0700166
167 if chrome_tool_repo.CommitLocally(commit_message):
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800168 self._logger.LogError(
169 'Commit to local branch "{0}" failed, aborted.'.format(
170 TEMP_BRANCH_NAME))
Han Shen91445322013-03-20 13:43:31 -0700171 return False
172 return True
173
174 def CheckoutBranch(self):
Han Shen819f8622014-04-08 16:57:38 -0700175 """Checkout working branch for the tools.
Han Shen91445322013-03-20 13:43:31 -0700176
Han Shen819f8622014-04-08 16:57:38 -0700177 Returns:
178 True: if operation succeeds.
179 """
Han Shen91445322013-03-20 13:43:31 -0700180
Han Shen819f8622014-04-08 16:57:38 -0700181 if self._gcc_branch:
182 rv = self.CheckoutToolBranch('gcc', self._gcc_branch)
183 if rv:
184 self._gcc_branch_tree = rv
185 else:
186 return False
187
188 if self._binutils_branch:
189 rv = self.CheckoutToolBranch('binutils', self._binutils_branch)
190 if rv:
191 self._binutils_branch_tree = rv
192 else:
193 return False
194
Han Shen91445322013-03-20 13:43:31 -0700195 return True
196
Han Shen819f8622014-04-08 16:57:38 -0700197 def CheckoutToolBranch(self, tool_name, tool_branch):
198 """Checkout the tool branch for a certain tool.
199
200 Args:
201 tool_name: either 'gcc' or 'binutils'
202 tool_branch: tool branch to use
203 Returns:
204 True: if operation succeeds. Otherwise False.
Han Shen91445322013-03-20 13:43:31 -0700205 """
Han Shen819f8622014-04-08 16:57:38 -0700206
207 chrome_tool_dir = self.GetChromeOsToolDir(tool_name)
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800208 command = 'cd "{0}" && git checkout {1}'.format(chrome_tool_dir,
209 tool_branch)
Han Shen819f8622014-04-08 16:57:38 -0700210 if not self._ce.RunCommand(command, print_to_console=True):
211 # Get 'TREE' value of this commit
212 command = ('cd "{0}" && git cat-file -p {1} '
213 '| grep -E "^tree [a-f0-9]+$" '
214 '| cut -d" " -f2').format(chrome_tool_dir, tool_branch)
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800215 ret, stdout, _ = self._ce.RunCommandWOutput(command,
216 print_to_console=False)
Han Shen819f8622014-04-08 16:57:38 -0700217 # Pipe operation always has a zero return value. So need to check if
218 # stdout is valid.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800219 if not ret and stdout and re.match('[0-9a-h]{40}', stdout.strip(),
220 re.IGNORECASE):
Han Shen819f8622014-04-08 16:57:38 -0700221 tool_branch_tree = stdout.strip()
222 self._logger.LogOutput('Find tree for {0} branch "{1}" - "{2}"'.format(
223 tool_name, tool_branch, tool_branch_tree))
224 return tool_branch_tree
225 self._logger.LogError(('Failed to checkout "{0}" or failed to '
226 'get tree value, aborted.').format(tool_branch))
227 return None
228
229 def FindEbuildFile(self):
230 """Find the ebuild files for the tools.
231
232 Returns:
233 True: if operation succeeds.
234 """
235
236 if self._gcc_branch:
237 (rv, ef, efn) = self.FindToolEbuildFile('gcc')
238 if rv:
239 self._gcc_ebuild_file = ef
240 self._gcc_ebuild_file_name = efn
241 else:
242 return False
243
244 if self._binutils_branch:
245 (rv, ef, efn) = self.FindToolEbuildFile('binutils')
246 if rv:
247 self._binutils_ebuild_file = ef
248 self._binutils_ebuild_file_name = efn
249 else:
250 return False
251
252 return True
253
254 def FindToolEbuildFile(self, tool_name):
255 """Find ebuild file for a specific tool.
256
257 Args:
258 tool_name: either "gcc" or "binutils".
259 Returns:
260 A triplet that consisits of whether operation succeeds or not,
261 tool ebuild file full path and tool ebuild file name.
262 """
263
264 # To get the active gcc ebuild file, we need a workable chroot first.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800265 if not os.path.exists(os.path.join(
266 self._chromeos_root, 'chroot')) and self._ce.RunCommand(
Han Shen819f8622014-04-08 16:57:38 -0700267 'cd "{0}" && cros_sdk --create'.format(self._chromeos_root)):
268 self._logger.LogError(('Failed to install a initial chroot, aborted.\n'
269 'If previous bootstrap failed, do a '
270 '"cros_sdk --delete" to remove '
271 'in-complete chroot.'))
272 return (False, None, None)
273
Luis Lozano036c9232015-12-10 10:47:01 -0800274 rv, stdout, _ = self._ce.ChrootRunCommandWOutput(
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800275 self._chromeos_root,
276 'equery w sys-devel/{0}'.format(tool_name),
Luis Lozano036c9232015-12-10 10:47:01 -0800277 print_to_console=True)
Han Shen819f8622014-04-08 16:57:38 -0700278 if rv:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800279 self._logger.LogError(('Failed to execute inside chroot '
280 '"equery w sys-devel/{0}", aborted.').format(
281 tool_name))
Han Shen819f8622014-04-08 16:57:38 -0700282 return (False, None, None)
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800283 m = re.match(r'^.*/({0}/(.*\.ebuild))$'.format(EBUILD_PATH_PATTERN.format(
284 tool_name)), stdout)
Han Shen819f8622014-04-08 16:57:38 -0700285 if not m:
286 self._logger.LogError(
287 ('Failed to find {0} ebuild file, aborted. '
288 'If previous bootstrap failed, do a "cros_sdk --delete" to remove '
289 'in-complete chroot.').format(tool_name))
290 return (False, None, None)
291 tool_ebuild_file = os.path.join(self._chromeos_root, m.group(1))
292 tool_ebuild_file_name = m.group(2)
293
294 return (True, tool_ebuild_file, tool_ebuild_file_name)
295
296 def InplaceModifyEbuildFile(self):
297 """Modify the ebuild file.
298
299 Returns:
300 True if operation succeeds.
301 """
302
303 # Note we shall not use remote branch name (eg. "cros/gcc.gnu.org/...") in
304 # CROS_WORKON_COMMIT, we have to use GITHASH. So we call GitGetCommitHash on
305 # tool_branch.
306 if self._gcc_branch:
307 tool_branch_githash = misc.GitGetCommitHash(
308 self.GetChromeOsToolDir('gcc'), self._gcc_branch)
309 if not tool_branch_githash:
310 return False
311 if not self.InplaceModifyToolEbuildFile(
312 tool_branch_githash, self._gcc_branch_tree, self._gcc_ebuild_file):
313 return False
314
315 if self._binutils_branch:
316 tool_branch_githash = misc.GitGetCommitHash(
317 self.GetChromeOsToolDir('binutils'), self._binutils_branch)
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800318 if not self.InplaceModifyToolEbuildFile(tool_branch_githash,
319 self._binutils_branch_tree,
320 self._binutils_ebuild_file):
Han Shen819f8622014-04-08 16:57:38 -0700321 return False
322 return True
323
324 @staticmethod
325 def ResetToolEbuildFile(chromeos_root, tool_name):
326 """Reset tool ebuild file to clean state.
327
328 Args:
329 chromeos_root: chromeos source tree
330 tool_name: either "gcc" or "binutils"
331 Returns:
332 True if operation succeds.
333 """
334 rv = misc.GetGitChangesAsList(
335 os.path.join(chromeos_root, CHROMIUMOS_OVERLAY_PATH),
336 path=('sys-devel/{0}/{0}-*.ebuild'.format(tool_name)),
337 staged=False)
338 if rv:
339 cmd = 'cd {0} && git checkout --'.format(os.path.join(
340 chromeos_root, CHROMIUMOS_OVERLAY_PATH))
341 for g in rv:
342 cmd += ' ' + g
343 rv = command_executer.GetCommandExecuter().RunCommand(cmd)
344 if rv:
345 logger.GetLogger().LogWarning(
346 'Failed to reset the ebuild file. Please refer to log above.')
347 return False
348 else:
349 logger.GetLogger().LogWarning(
350 'Note - did not find any modified {0} ebuild file.'.format(tool_name))
351 # Fall through
352 return True
353
354 def GetChromeOsToolDir(self, tool_name):
355 """Return the chromeos git dir for a specific tool.
356
357 Args:
358 tool_name: either 'gcc' or 'binutils'.
359 Returns:
360 Absolute git path for the tool.
361 """
362
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800363 return os.path.join(self._chromeos_root,
364 REPO_PATH_PATTERN.format(tool_name))
Han Shen819f8622014-04-08 16:57:38 -0700365
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800366 def InplaceModifyToolEbuildFile(self, tool_branch_githash, tool_branch_tree,
367 tool_ebuild_file):
Han Shen819f8622014-04-08 16:57:38 -0700368 """Using sed to fill properly values into the ebuild file.
369
370 Args:
371 tool_branch_githash: githash for tool_branch
372 tool_branch_tree: treeish for the tool branch
373 tool_ebuild_file: tool ebuild file
374 Returns:
375 True: if operation succeeded.
376 """
377
378 command = ('sed -i '
379 '-e \'/^CROS_WORKON_COMMIT=".*"/i'
380 ' # The following line is modified by script.\' '
381 '-e \'s!^CROS_WORKON_COMMIT=".*"$!CROS_WORKON_COMMIT="{0}"!\' '
382 '-e \'/^CROS_WORKON_TREE=".*"/i'
383 ' # The following line is modified by script.\' '
384 '-e \'s!^CROS_WORKON_TREE=".*"$!CROS_WORKON_TREE="{1}"!\' '
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800385 '{2}').format(tool_branch_githash, tool_branch_tree,
Han Shen819f8622014-04-08 16:57:38 -0700386 tool_ebuild_file)
Han Shen91445322013-03-20 13:43:31 -0700387 rv = self._ce.RunCommand(command)
388 if rv:
389 self._logger.LogError(
Han Shen819f8622014-04-08 16:57:38 -0700390 'Failed to modify commit and tree value for "{0}"", aborted.'.format(
391 tool_ebuild_file))
392 return False
393
394 # Warn that the ebuild file has been modified.
395 self._logger.LogWarning(
396 ('Ebuild file "{0}" is modified, to revert the file - \n'
397 'bootstrap_compiler.py --chromeos_root={1} '
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800398 '--reset_tool_ebuild_file').format(tool_ebuild_file,
399 self._chromeos_root))
Han Shen819f8622014-04-08 16:57:38 -0700400 return True
401
402 def DoBuildForBoard(self):
403 """Build tool for a specific board.
404
405 Returns:
406 True if operation succeeds.
407 """
408
409 if self._gcc_branch:
410 if not self.DoBuildToolForBoard('gcc'):
411 return False
412 if self._binutils_branch:
413 if not self.DoBuildToolForBoard('binutils'):
414 return False
415 return True
416
417 def DoBuildToolForBoard(self, tool_name):
418 """Build a specific tool for a specific board.
419
420 Args:
421 tool_name: either "gcc" or "binutils"
422 Returns:
423 True if operation succeeds.
424 """
425
Han Shenfe3001c2014-04-28 16:36:28 -0700426 boards_to_build = self._board.split(',')
Han Shen819f8622014-04-08 16:57:38 -0700427
Han Shenfe3001c2014-04-28 16:36:28 -0700428 failed = []
429 for board in boards_to_build:
430 if board == 'host':
431 command = 'sudo emerge sys-devel/{0}'.format(tool_name)
432 else:
433 target = misc.GetCtargetFromBoard(board, self._chromeos_root)
434 if not target:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800435 self._logger.LogError('Unsupported board "{0}", skip.'.format(board))
Han Shenfe3001c2014-04-28 16:36:28 -0700436 failed.append(board)
437 continue
438 command = 'sudo emerge cross-{0}/{1}'.format(target, tool_name)
439
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800440 rv = self._ce.ChrootRunCommand(self._chromeos_root,
441 command,
Luis Lozano036c9232015-12-10 10:47:01 -0800442 print_to_console=True)
Han Shenfe3001c2014-04-28 16:36:28 -0700443 if rv:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800444 self._logger.LogError('Build {0} failed for {1}, aborted.'.format(
445 tool_name, board))
Han Shenfe3001c2014-04-28 16:36:28 -0700446 failed.append(board)
447 else:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800448 self._logger.LogOutput('Successfully built {0} for board {1}.'.format(
449 tool_name, board))
Han Shenfe3001c2014-04-28 16:36:28 -0700450
Han Shen8dfdabd2014-05-08 10:15:09 -0700451 if failed:
Han Shen819f8622014-04-08 16:57:38 -0700452 self._logger.LogError(
Han Shenfe3001c2014-04-28 16:36:28 -0700453 'Failed to build {0} for the following board(s): "{1}"'.format(
454 tool_name, ' '.join(failed)))
Han Shen91445322013-03-20 13:43:31 -0700455 return False
Han Shenfe3001c2014-04-28 16:36:28 -0700456 # All boards build successfully
Han Shen91445322013-03-20 13:43:31 -0700457 return True
458
459 def DoBootstrapping(self):
Han Shen819f8622014-04-08 16:57:38 -0700460 """Do bootstrapping the chroot.
461
Han Shen03d30982014-06-12 11:22:29 -0700462 This step firstly downloads a prestine sdk, then use this sdk to build the
463 new sdk, finally use the new sdk to build every host package.
464
Han Shen819f8622014-04-08 16:57:38 -0700465 Returns:
466 True if operation succeeds.
467 """
468
Han Shen91445322013-03-20 13:43:31 -0700469 logfile = os.path.join(self._chromeos_root, 'bootstrap.log')
Han Shen819f8622014-04-08 16:57:38 -0700470 command = 'cd "{0}" && cros_sdk --delete --bootstrap |& tee "{1}"'.format(
471 self._chromeos_root, logfile)
Luis Lozano036c9232015-12-10 10:47:01 -0800472 rv = self._ce.RunCommand(command, print_to_console=True)
Han Shen91445322013-03-20 13:43:31 -0700473 if rv:
474 self._logger.LogError('Bootstrapping failed, log file - "{0}"\n'.format(
475 logfile))
476 return False
477
478 self._logger.LogOutput('Bootstrap succeeded.')
479 return True
480
Han Shen03d30982014-06-12 11:22:29 -0700481 def BuildAndInstallAmd64Host(self):
482 """Build amd64-host (host) packages.
483
484 Build all host packages in the newly-bootstrapped 'chroot' using *NEW*
485 toolchain.
486
487 So actually we perform 2 builds of all host packages -
488 1. build new toolchain using old toolchain and build all host packages
489 using the newly built toolchain
490 2. build the new toolchain again but using new toolchain built in step 1,
491 and build all host packages using the newly built toolchain
492
493 Returns:
494 True if operation succeeds.
495 """
496
497 cmd = ('cd {0} && cros_sdk -- -- ./setup_board --board=amd64-host '
498 '--accept_licenses=@CHROMEOS --skip_chroot_upgrade --nousepkg '
499 '--reuse_pkgs_from_local_boards').format(self._chromeos_root)
Luis Lozano036c9232015-12-10 10:47:01 -0800500 rv = self._ce.RunCommand(cmd, print_to_console=True)
Han Shen03d30982014-06-12 11:22:29 -0700501 if rv:
502 self._logger.LogError('Build amd64-host failed.')
503 return False
504
505 # Package amd64-host into 'built-sdk.tar.xz'.
506 sdk_package = os.path.join(self._chromeos_root, 'built-sdk.tar.xz')
507 cmd = ('cd {0}/chroot/build/amd64-host && sudo XZ_OPT="-e9" '
508 'tar --exclude="usr/lib/debug/*" --exclude="packages/*" '
509 '--exclude="tmp/*" --exclude="usr/local/build/autotest/*" '
510 '--sparse -I xz -vcf {1} . && sudo chmod a+r {1}').format(
511 self._chromeos_root, sdk_package)
Luis Lozano036c9232015-12-10 10:47:01 -0800512 rv = self._ce.RunCommand(cmd, print_to_console=True)
Han Shen03d30982014-06-12 11:22:29 -0700513 if rv:
514 self._logger.LogError('Failed to create "built-sdk.tar.xz".')
515 return False
516
517 # Install amd64-host into a new chroot.
518 cmd = ('cd {0} && cros_sdk --chroot new-sdk-chroot --download --replace '
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800519 '--nousepkg --url file://{1}').format(self._chromeos_root,
520 sdk_package)
Luis Lozano036c9232015-12-10 10:47:01 -0800521 rv = self._ce.RunCommand(cmd, print_to_console=True)
Han Shen03d30982014-06-12 11:22:29 -0700522 if rv:
523 self._logger.LogError('Failed to install "built-sdk.tar.xz".')
524 return False
525 self._logger.LogOutput(
526 'Successfully installed built-sdk.tar.xz into a new chroot.\nAll done.')
527
528 # Rename the newly created new-sdk-chroot to chroot.
529 cmd = ('cd {0} && sudo mv chroot chroot-old && '
530 'sudo mv new-sdk-chroot chroot').format(self._chromeos_root)
Luis Lozano036c9232015-12-10 10:47:01 -0800531 rv = self._ce.RunCommand(cmd, print_to_console=True)
Han Shen03d30982014-06-12 11:22:29 -0700532 return rv == 0
533
Han Shen91445322013-03-20 13:43:31 -0700534 def Do(self):
Han Shen819f8622014-04-08 16:57:38 -0700535 """Entrance of the class.
536
537 Returns:
538 True if everything is ok.
539 """
540
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800541 if (self.SubmitToLocalBranch() and self.CheckoutBranch() and
542 self.FindEbuildFile() and self.InplaceModifyEbuildFile()):
Han Shen819f8622014-04-08 16:57:38 -0700543 if self._setup_tool_ebuild_file_only:
544 # Everything is done, we are good.
545 ret = True
546 else:
547 if self._board:
548 ret = self.DoBuildForBoard()
549 else:
550 # This implies '--bootstrap'.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800551 ret = (self.DoBootstrapping() and (self._disable_2nd_bootstrap or
552 self.BuildAndInstallAmd64Host()))
Han Shen91445322013-03-20 13:43:31 -0700553 else:
554 ret = False
Han Shen91445322013-03-20 13:43:31 -0700555 return ret
556
557
558def Main(argv):
559 parser = optparse.OptionParser()
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800560 parser.add_option('-c',
561 '--chromeos_root',
562 dest='chromeos_root',
Han Shen819f8622014-04-08 16:57:38 -0700563 help=('Optional. ChromeOs root dir. '
564 'When not specified, chromeos root will be deduced '
565 'from current working directory.'))
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800566 parser.add_option('--gcc_branch',
567 dest='gcc_branch',
Han Shen91445322013-03-20 13:43:31 -0700568 help=('The branch to test against. '
569 'This branch must be a local branch '
570 'inside "src/third_party/gcc". '
Han Shen819f8622014-04-08 16:57:38 -0700571 'Notice, this must not be used with "--gcc_dir".'))
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800572 parser.add_option('--binutils_branch',
573 dest='binutils_branch',
Han Shen819f8622014-04-08 16:57:38 -0700574 help=('The branch to test against binutils. '
575 'This branch must be a local branch '
576 'inside "src/third_party/binutils". '
577 'Notice, this must not be used with '
578 '"--binutils_dir".'))
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800579 parser.add_option('-g',
580 '--gcc_dir',
581 dest='gcc_dir',
Han Shen91445322013-03-20 13:43:31 -0700582 help=('Use a local gcc tree to do bootstrapping. '
Han Shen819f8622014-04-08 16:57:38 -0700583 'Notice, this must not be used with "--gcc_branch".'))
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800584 parser.add_option('--binutils_dir',
585 dest='binutils_dir',
Han Shen819f8622014-04-08 16:57:38 -0700586 help=('Use a local binutils tree to do bootstrapping. '
587 'Notice, this must not be used with '
588 '"--binutils_branch".'))
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800589 parser.add_option('--fixperm',
590 dest='fixperm',
591 default=False,
592 action='store_true',
Han Shen91445322013-03-20 13:43:31 -0700593 help=('Fix the (notorious) permission error '
594 'while trying to bootstrap the chroot. '
595 'Note this takes an extra 10-15 minutes '
596 'and is only needed once per chromiumos tree.'))
Han Shen819f8622014-04-08 16:57:38 -0700597 parser.add_option('--setup_tool_ebuild_file_only',
598 dest='setup_tool_ebuild_file_only',
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800599 default=False,
600 action='store_true',
Han Shen819f8622014-04-08 16:57:38 -0700601 help=('Setup gcc and/or binutils ebuild file '
602 'to pick up the branch (--gcc/binutils_branch) or '
603 'use gcc and/or binutils source (--gcc/binutils_dir) '
604 'and exit. Keep chroot as is. This should not be '
605 'used with --gcc/binutils_dir/branch options.'))
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800606 parser.add_option('--reset_tool_ebuild_file',
607 dest='reset_tool_ebuild_file',
608 default=False,
609 action='store_true',
Han Shen91445322013-03-20 13:43:31 -0700610 help=('Reset the modification that is done by this script.'
611 'Note, when this script is running, it will modify '
Han Shen819f8622014-04-08 16:57:38 -0700612 'the active gcc/binutils ebuild file. Use this '
613 'option to reset (what this script has done) '
614 'and exit. This should not be used with -- '
615 'gcc/binutils_dir/branch options.'))
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800616 parser.add_option('--board',
617 dest='board',
618 default=None,
Han Shenfe3001c2014-04-28 16:36:28 -0700619 help=('Only build toolchain for specific board(s). '
Han Shen819f8622014-04-08 16:57:38 -0700620 'Use "host" to build for host. '
Han Shenfe3001c2014-04-28 16:36:28 -0700621 'Use "," to seperate multiple boards. '
Han Shen819f8622014-04-08 16:57:38 -0700622 'This does not perform a chroot bootstrap.'))
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800623 parser.add_option('--bootstrap',
624 dest='bootstrap',
625 default=False,
626 action='store_true',
Han Shen819f8622014-04-08 16:57:38 -0700627 help=('Performs a chroot bootstrap. '
628 'Note, this will *destroy* your current chroot.'))
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800629 parser.add_option('--disable-2nd-bootstrap',
630 dest='disable_2nd_bootstrap',
631 default=False,
632 action='store_true',
Han Shen03d30982014-06-12 11:22:29 -0700633 help=('Disable a second bootstrap '
634 '(build of amd64-host stage).'))
Han Shen819f8622014-04-08 16:57:38 -0700635
Han Shen91445322013-03-20 13:43:31 -0700636 options = parser.parse_args(argv)[0]
Han Shen819f8622014-04-08 16:57:38 -0700637 # Trying to deduce chromeos root from current directory.
Han Shen91445322013-03-20 13:43:31 -0700638 if not options.chromeos_root:
Han Shen819f8622014-04-08 16:57:38 -0700639 logger.GetLogger().LogOutput('Trying to deduce chromeos root ...')
640 wdir = os.getcwd()
641 while wdir and wdir != '/':
642 if misc.IsChromeOsTree(wdir):
643 logger.GetLogger().LogOutput('Find chromeos_root: {}'.format(wdir))
644 options.chromeos_root = wdir
645 break
646 wdir = os.path.dirname(wdir)
647
648 if not options.chromeos_root:
649 parser.error('Missing or failing to deduce mandatory option "--chromeos".')
Han Shen91445322013-03-20 13:43:31 -0700650 return 1
651
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800652 options.chromeos_root = os.path.abspath(os.path.expanduser(
653 options.chromeos_root))
Han Shen91445322013-03-20 13:43:31 -0700654
655 if not os.path.isdir(options.chromeos_root):
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800656 logger.GetLogger().LogError('"{0}" does not exist.'.format(
657 options.chromeos_root))
Han Shen91445322013-03-20 13:43:31 -0700658 return 1
659
660 if options.fixperm:
661 # Fix perm error before continuing.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800662 cmd = (
663 r'sudo find "{0}" \( -name ".cache" -type d -prune \) -o '
664 r'\( -name "chroot" -type d -prune \) -o '
665 r'\( -type f -exec chmod a+r {{}} \; \) -o '
666 r'\( -type d -exec chmod a+rx {{}} \; \)').format(options.chromeos_root)
Han Shen91445322013-03-20 13:43:31 -0700667 logger.GetLogger().LogOutput(
Han Shen819f8622014-04-08 16:57:38 -0700668 'Fixing perm issues for chromeos root, this might take some time.')
Han Shen91445322013-03-20 13:43:31 -0700669 command_executer.GetCommandExecuter().RunCommand(cmd)
670
Han Shen819f8622014-04-08 16:57:38 -0700671 if options.reset_tool_ebuild_file:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800672 if (options.gcc_dir or options.gcc_branch or options.binutils_dir or
673 options.binutils_branch):
Han Shen91445322013-03-20 13:43:31 -0700674 logger.GetLogger().LogWarning(
Han Shen819f8622014-04-08 16:57:38 -0700675 'Ignoring any "--gcc/binutils_dir" and/or "--gcc/binutils_branch".')
676 if options.setup_tool_ebuild_file_only:
677 logger.GetLogger().LogError(
678 ('Conflict options "--reset_tool_ebuild_file" '
679 'and "--setup_tool_ebuild_file_only".'))
Han Shen91445322013-03-20 13:43:31 -0700680 return 1
Han Shen819f8622014-04-08 16:57:38 -0700681 rv = Bootstrapper.ResetToolEbuildFile(options.chromeos_root, 'gcc')
682 rv1 = Bootstrapper.ResetToolEbuildFile(options.chromeos_root, 'binutils')
683 return 0 if (rv and rv1) else 1
Han Shen91445322013-03-20 13:43:31 -0700684
685 if options.gcc_dir:
686 options.gcc_dir = os.path.abspath(os.path.expanduser(options.gcc_dir))
687 if not os.path.isdir(options.gcc_dir):
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800688 logger.GetLogger().LogError('"{0}" does not exist.'.format(
689 options.gcc_dir))
Han Shen91445322013-03-20 13:43:31 -0700690 return 1
691
Han Shen819f8622014-04-08 16:57:38 -0700692 if options.gcc_branch and options.gcc_dir:
693 parser.error('Only one of "--gcc_dir" and "--gcc_branch" can be specified.')
Han Shen91445322013-03-20 13:43:31 -0700694 return 1
Han Shen819f8622014-04-08 16:57:38 -0700695
696 if options.binutils_dir:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800697 options.binutils_dir = os.path.abspath(os.path.expanduser(
698 options.binutils_dir))
Han Shen819f8622014-04-08 16:57:38 -0700699 if not os.path.isdir(options.binutils_dir):
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800700 logger.GetLogger().LogError('"{0}" does not exist.'.format(
701 options.binutils_dir))
Han Shen819f8622014-04-08 16:57:38 -0700702 return 1
703
704 if options.binutils_branch and options.binutils_dir:
705 parser.error('Only one of "--binutils_dir" and '
706 '"--binutils_branch" can be specified.')
707 return 1
708
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800709 if (not (options.binutils_branch or options.binutils_dir or options.gcc_branch
710 or options.gcc_dir)):
Han Shen819f8622014-04-08 16:57:38 -0700711 parser.error(('At least one of "--gcc_dir", "--gcc_branch", '
712 '"--binutils_dir" and "--binutils_branch" must '
713 'be specified.'))
714 return 1
715
716 if not options.board and not options.bootstrap:
717 parser.error('You must specify either "--board" or "--bootstrap".')
718 return 1
719
Han Shen03d30982014-06-12 11:22:29 -0700720 if (options.board and options.bootstrap and
721 not options.setup_tool_ebuild_file_only):
Han Shen819f8622014-04-08 16:57:38 -0700722 parser.error('You must specify only one of "--board" and "--bootstrap".')
Han Shen91445322013-03-20 13:43:31 -0700723 return 1
724
Han Shen03d30982014-06-12 11:22:29 -0700725 if not options.bootstrap and options.disable_2nd_bootstrap:
726 parser.error('"--disable-2nd-bootstrap" has no effect '
727 'without specifying "--bootstrap".')
728 return 1
729
Han Shen91445322013-03-20 13:43:31 -0700730 if Bootstrapper(
Han Shen819f8622014-04-08 16:57:38 -0700731 options.chromeos_root,
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800732 gcc_branch=options.gcc_branch,
733 gcc_dir=options.gcc_dir,
Han Shen819f8622014-04-08 16:57:38 -0700734 binutils_branch=options.binutils_branch,
735 binutils_dir=options.binutils_dir,
736 board=options.board,
Han Shen03d30982014-06-12 11:22:29 -0700737 disable_2nd_bootstrap=options.disable_2nd_bootstrap,
Han Shen819f8622014-04-08 16:57:38 -0700738 setup_tool_ebuild_file_only=options.setup_tool_ebuild_file_only).Do():
Han Shen91445322013-03-20 13:43:31 -0700739 return 0
740 return 1
741
742
743if __name__ == '__main__':
744 retval = Main(sys.argv)
745 sys.exit(retval)