blob: 992c03efc75c4dc3c095136ec30e7e10cee01181 [file] [log] [blame]
kjellander@webrtc.org89256622014-08-20 12:10:11 +00001#!/usr/bin/env python
2# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS. All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
9
10"""Setup links to a Chromium checkout for WebRTC.
11
12WebRTC standalone shares a lot of dependencies and build tools with Chromium.
13To do this, many of the paths of a Chromium checkout is emulated by creating
14symlinks to files and directories. This script handles the setup of symlinks to
15achieve this.
16
17It also handles cleanup of the legacy Subversion-based approach that was used
18before Chrome switched over their master repo from Subversion to Git.
19"""
20
21
22import ctypes
23import errno
24import logging
25import optparse
26import os
27import shelve
28import shutil
29import subprocess
30import sys
31import textwrap
32
33
34DIRECTORIES = [
35 'build',
36 'buildtools',
37 'google_apis', # Needed by build/common.gypi.
38 'net',
39 'testing',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000040 'third_party/binutils',
41 'third_party/boringssl',
42 'third_party/colorama',
43 'third_party/drmemory',
44 'third_party/expat',
45 'third_party/icu',
kjellander@webrtc.org4e4fe4f2014-10-01 08:03:19 +000046 'third_party/instrumented_libraries',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000047 'third_party/jsoncpp',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000048 'third_party/libjpeg',
49 'third_party/libjpeg_turbo',
50 'third_party/libsrtp',
kjellander@webrtc.org7d4e6d02014-11-27 10:41:04 +000051 'third_party/libudev',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000052 'third_party/libvpx',
53 'third_party/libyuv',
54 'third_party/llvm-build',
55 'third_party/nss',
tkchin@webrtc.org3a63a3c2015-01-06 07:21:34 +000056 'third_party/ocmock',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000057 'third_party/openmax_dl',
58 'third_party/opus',
59 'third_party/protobuf',
60 'third_party/sqlite',
61 'third_party/syzygy',
62 'third_party/usrsctp',
63 'third_party/yasm',
kjellander@webrtc.org3bd41562014-09-01 11:06:37 +000064 'third_party/zlib',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000065 'tools/clang',
66 'tools/generate_library_loader',
67 'tools/gn',
68 'tools/gyp',
69 'tools/memory',
70 'tools/protoc_wrapper',
71 'tools/python',
72 'tools/swarming_client',
73 'tools/valgrind',
Andrew MacDonald65de7d22015-05-19 11:37:34 -070074 'tools/vim',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000075 'tools/win',
76]
77
kjellander@webrtc.org3bd41562014-09-01 11:06:37 +000078from sync_chromium import get_target_os_list
79if 'android' in get_target_os_list():
80 DIRECTORIES += [
81 'base',
82 'third_party/android_testrunner',
83 'third_party/android_tools',
kjellander@webrtc.orgcbe7ca82015-01-06 07:24:27 +000084 'third_party/appurify-python',
kjellander@webrtc.orgb8caf6a2014-09-30 18:05:02 +000085 'third_party/ashmem',
86 'third_party/jsr-305',
Henrik Kjellander10ba3ee2015-04-29 14:47:53 +020087 'third_party/junit',
kjellander@webrtc.orgb8caf6a2014-09-30 18:05:02 +000088 'third_party/libevent',
89 'third_party/libxml',
Henrik Kjellander10ba3ee2015-04-29 14:47:53 +020090 'third_party/mockito',
kjellander@webrtc.orgb8caf6a2014-09-30 18:05:02 +000091 'third_party/modp_b64',
kjellander@webrtc.orgcbe7ca82015-01-06 07:24:27 +000092 'third_party/requests',
Henrik Kjellander10ba3ee2015-04-29 14:47:53 +020093 'third_party/robolectric',
kjellander@webrtc.org3bd41562014-09-01 11:06:37 +000094 'tools/android',
kjellander@webrtc.orgbfdee692015-02-03 15:23:34 +000095 'tools/grit',
marpan@webrtc.org4765ca52014-11-03 20:10:26 +000096 'tools/relocation_packer'
kjellander@webrtc.org3bd41562014-09-01 11:06:37 +000097 ]
98
kjellander@webrtc.org89256622014-08-20 12:10:11 +000099FILES = {
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000100 'tools/find_depot_tools.py': None,
101 'third_party/BUILD.gn': None,
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000102}
103
kjellander@webrtc.orge94f83a2014-09-18 13:47:23 +0000104ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000105CHROMIUM_CHECKOUT = os.path.join('chromium', 'src')
106LINKS_DB = 'links'
107
108# Version management to make future upgrades/downgrades easier to support.
109SCHEMA_VERSION = 1
110
111
112def query_yes_no(question, default=False):
113 """Ask a yes/no question via raw_input() and return their answer.
114
115 Modified from http://stackoverflow.com/a/3041990.
116 """
117 prompt = " [%s/%%s]: "
118 prompt = prompt % ('Y' if default is True else 'y')
119 prompt = prompt % ('N' if default is False else 'n')
120
121 if default is None:
122 default = 'INVALID'
123
124 while True:
125 sys.stdout.write(question + prompt)
126 choice = raw_input().lower()
127 if choice == '' and default != 'INVALID':
128 return default
129
130 if 'yes'.startswith(choice):
131 return True
132 elif 'no'.startswith(choice):
133 return False
134
135 print "Please respond with 'yes' or 'no' (or 'y' or 'n')."
136
137
138# Actions
139class Action(object):
140 def __init__(self, dangerous):
141 self.dangerous = dangerous
142
143 def announce(self, planning):
144 """Log a description of this action.
145
146 Args:
147 planning - True iff we're in the planning stage, False if we're in the
148 doit stage.
149 """
150 pass
151
152 def doit(self, links_db):
153 """Execute the action, recording what we did to links_db, if necessary."""
154 pass
155
156
157class Remove(Action):
158 def __init__(self, path, dangerous):
159 super(Remove, self).__init__(dangerous)
160 self._priority = 0
161 self._path = path
162
163 def announce(self, planning):
164 log = logging.warn
165 filesystem_type = 'file'
166 if not self.dangerous:
167 log = logging.info
168 filesystem_type = 'link'
169 if planning:
170 log('Planning to remove %s: %s', filesystem_type, self._path)
171 else:
172 log('Removing %s: %s', filesystem_type, self._path)
173
Henrik Kjellander57e5fd22015-05-25 12:55:39 +0200174 def doit(self, _):
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000175 os.remove(self._path)
176
177
178class Rmtree(Action):
179 def __init__(self, path):
180 super(Rmtree, self).__init__(dangerous=True)
181 self._priority = 0
182 self._path = path
183
184 def announce(self, planning):
185 if planning:
186 logging.warn('Planning to remove directory: %s', self._path)
187 else:
188 logging.warn('Removing directory: %s', self._path)
189
Henrik Kjellander57e5fd22015-05-25 12:55:39 +0200190 def doit(self, _):
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000191 if sys.platform.startswith('win'):
192 # shutil.rmtree() doesn't work on Windows if any of the directories are
193 # read-only, which svn repositories are.
194 subprocess.check_call(['rd', '/q', '/s', self._path], shell=True)
195 else:
196 shutil.rmtree(self._path)
197
198
199class Makedirs(Action):
200 def __init__(self, path):
201 super(Makedirs, self).__init__(dangerous=False)
202 self._priority = 1
203 self._path = path
204
Henrik Kjellander57e5fd22015-05-25 12:55:39 +0200205 def doit(self, _):
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000206 try:
207 os.makedirs(self._path)
208 except OSError as e:
209 if e.errno != errno.EEXIST:
210 raise
211
212
213class Symlink(Action):
214 def __init__(self, source_path, link_path):
215 super(Symlink, self).__init__(dangerous=False)
216 self._priority = 2
217 self._source_path = source_path
218 self._link_path = link_path
219
220 def announce(self, planning):
221 if planning:
222 logging.info(
223 'Planning to create link from %s to %s', self._link_path,
224 self._source_path)
225 else:
226 logging.debug(
227 'Linking from %s to %s', self._link_path, self._source_path)
228
229 def doit(self, links_db):
230 # Files not in the root directory need relative path calculation.
231 # On Windows, use absolute paths instead since NTFS doesn't seem to support
232 # relative paths for symlinks.
233 if sys.platform.startswith('win'):
234 source_path = os.path.abspath(self._source_path)
235 else:
236 if os.path.dirname(self._link_path) != self._link_path:
237 source_path = os.path.relpath(self._source_path,
238 os.path.dirname(self._link_path))
239
240 os.symlink(source_path, os.path.abspath(self._link_path))
241 links_db[self._source_path] = self._link_path
242
243
244class LinkError(IOError):
245 """Failed to create a link."""
246 pass
247
248
249# Handles symlink creation on the different platforms.
250if sys.platform.startswith('win'):
251 def symlink(source_path, link_path):
252 flag = 1 if os.path.isdir(source_path) else 0
253 if not ctypes.windll.kernel32.CreateSymbolicLinkW(
254 unicode(link_path), unicode(source_path), flag):
255 raise OSError('Failed to create symlink to %s. Notice that only NTFS '
256 'version 5.0 and up has all the needed APIs for '
257 'creating symlinks.' % source_path)
258 os.symlink = symlink
259
260
Henrik Kjellander57e5fd22015-05-25 12:55:39 +0200261class WebRTCLinkSetup(object):
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000262 def __init__(self, links_db, force=False, dry_run=False, prompt=False):
263 self._force = force
264 self._dry_run = dry_run
265 self._prompt = prompt
266 self._links_db = links_db
267
268 def CreateLinks(self, on_bot):
269 logging.debug('CreateLinks')
270 # First, make a plan of action
271 actions = []
272
273 for source_path, link_path in FILES.iteritems():
274 actions += self._ActionForPath(
275 source_path, link_path, check_fn=os.path.isfile, check_msg='files')
276 for source_dir in DIRECTORIES:
277 actions += self._ActionForPath(
278 source_dir, None, check_fn=os.path.isdir,
279 check_msg='directories')
280
kjellander@webrtc.orge94f83a2014-09-18 13:47:23 +0000281 if not on_bot and self._force:
282 # When making the manual switch from legacy SVN checkouts to the new
283 # Git-based Chromium DEPS, the .gclient_entries file that contains cached
284 # URLs for all DEPS entries must be removed to avoid future sync problems.
285 entries_file = os.path.join(os.path.dirname(ROOT_DIR), '.gclient_entries')
286 if os.path.exists(entries_file):
287 actions.append(Remove(entries_file, dangerous=True))
288
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000289 actions.sort()
290
291 if self._dry_run:
292 for action in actions:
293 action.announce(planning=True)
294 logging.info('Not doing anything because dry-run was specified.')
295 sys.exit(0)
296
297 if any(a.dangerous for a in actions):
298 logging.warn('Dangerous actions:')
299 for action in (a for a in actions if a.dangerous):
300 action.announce(planning=True)
301 print
302
303 if not self._force:
304 logging.error(textwrap.dedent("""\
305 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
306 A C T I O N R E Q I R E D
307 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
308
309 Because chromium/src is transitioning to Git (from SVN), we needed to
310 change the way that the WebRTC standalone checkout works. Instead of
311 individually syncing subdirectories of Chromium in SVN, we're now
312 syncing Chromium (and all of its DEPS, as defined by its own DEPS file),
313 into the `chromium/src` directory.
314
315 As such, all Chromium directories which are currently pulled by DEPS are
316 now replaced with a symlink into the full Chromium checkout.
317
318 To avoid disrupting developers, we've chosen to not delete your
319 directories forcibly, in case you have some work in progress in one of
320 them :).
321
322 ACTION REQUIRED:
323 Before running `gclient sync|runhooks` again, you must run:
324 %s%s --force
325
326 Which will replace all directories which now must be symlinks, after
327 prompting with a summary of the work-to-be-done.
328 """), 'python ' if sys.platform.startswith('win') else '', sys.argv[0])
329 sys.exit(1)
330 elif self._prompt:
331 if not query_yes_no('Would you like to perform the above plan?'):
332 sys.exit(1)
333
334 for action in actions:
335 action.announce(planning=False)
336 action.doit(self._links_db)
337
338 if not on_bot and self._force:
339 logging.info('Completed!\n\nNow run `gclient sync|runhooks` again to '
340 'let the remaining hooks (that probably were interrupted) '
341 'execute.')
342
343 def CleanupLinks(self):
344 logging.debug('CleanupLinks')
345 for source, link_path in self._links_db.iteritems():
346 if source == 'SCHEMA_VERSION':
347 continue
348 if os.path.islink(link_path) or sys.platform.startswith('win'):
349 # os.path.islink() always returns false on Windows
350 # See http://bugs.python.org/issue13143.
351 logging.debug('Removing link to %s at %s', source, link_path)
352 if not self._dry_run:
353 if os.path.exists(link_path):
354 if sys.platform.startswith('win') and os.path.isdir(link_path):
Henrik Kjellanderc444de62015-04-29 11:27:22 +0200355 subprocess.check_call(['rmdir', '/q', '/s', link_path],
356 shell=True)
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000357 else:
358 os.remove(link_path)
359 del self._links_db[source]
360
361 @staticmethod
362 def _ActionForPath(source_path, link_path=None, check_fn=None,
363 check_msg=None):
364 """Create zero or more Actions to link to a file or directory.
365
366 This will be a symlink on POSIX platforms. On Windows this requires
367 that NTFS is version 5.0 or higher (Vista or newer).
368
369 Args:
370 source_path: Path relative to the Chromium checkout root.
371 For readability, the path may contain slashes, which will
372 automatically be converted to the right path delimiter on Windows.
373 link_path: The location for the link to create. If omitted it will be the
374 same path as source_path.
375 check_fn: A function returning true if the type of filesystem object is
376 correct for the attempted call. Otherwise an error message with
377 check_msg will be printed.
378 check_msg: String used to inform the user of an invalid attempt to create
379 a file.
380 Returns:
381 A list of Action objects.
382 """
383 def fix_separators(path):
384 if sys.platform.startswith('win'):
385 return path.replace(os.altsep, os.sep)
386 else:
387 return path
388
389 assert check_fn
390 assert check_msg
391 link_path = link_path or source_path
392 link_path = fix_separators(link_path)
393
394 source_path = fix_separators(source_path)
395 source_path = os.path.join(CHROMIUM_CHECKOUT, source_path)
396 if os.path.exists(source_path) and not check_fn:
397 raise LinkError('_LinkChromiumPath can only be used to link to %s: '
398 'Tried to link to: %s' % (check_msg, source_path))
399
400 if not os.path.exists(source_path):
401 logging.debug('Silently ignoring missing source: %s. This is to avoid '
402 'errors on platform-specific dependencies.', source_path)
403 return []
404
405 actions = []
406
407 if os.path.exists(link_path) or os.path.islink(link_path):
408 if os.path.islink(link_path):
409 actions.append(Remove(link_path, dangerous=False))
410 elif os.path.isfile(link_path):
411 actions.append(Remove(link_path, dangerous=True))
412 elif os.path.isdir(link_path):
413 actions.append(Rmtree(link_path))
414 else:
415 raise LinkError('Don\'t know how to plan: %s' % link_path)
416
417 # Create parent directories to the target link if needed.
418 target_parent_dirs = os.path.dirname(link_path)
419 if (target_parent_dirs and
420 target_parent_dirs != link_path and
421 not os.path.exists(target_parent_dirs)):
422 actions.append(Makedirs(target_parent_dirs))
423
424 actions.append(Symlink(source_path, link_path))
425
426 return actions
427
428def _initialize_database(filename):
429 links_database = shelve.open(filename)
430
431 # Wipe the database if this version of the script ends up looking at a
432 # newer (future) version of the links db, just to be sure.
433 version = links_database.get('SCHEMA_VERSION')
434 if version and version != SCHEMA_VERSION:
435 logging.info('Found database with schema version %s while this script only '
436 'supports %s. Wiping previous database contents.', version,
437 SCHEMA_VERSION)
438 links_database.clear()
439 links_database['SCHEMA_VERSION'] = SCHEMA_VERSION
440 return links_database
441
442
443def main():
444 on_bot = os.environ.get('CHROME_HEADLESS') == '1'
445
446 parser = optparse.OptionParser()
447 parser.add_option('-d', '--dry-run', action='store_true', default=False,
448 help='Print what would be done, but don\'t perform any '
449 'operations. This will automatically set logging to '
450 'verbose.')
451 parser.add_option('-c', '--clean-only', action='store_true', default=False,
452 help='Only clean previously created links, don\'t create '
453 'new ones. This will automatically set logging to '
454 'verbose.')
455 parser.add_option('-f', '--force', action='store_true', default=on_bot,
456 help='Force link creation. CAUTION: This deletes existing '
457 'folders and files in the locations where links are '
458 'about to be created.')
459 parser.add_option('-n', '--no-prompt', action='store_false', dest='prompt',
460 default=(not on_bot),
461 help='Prompt if we\'re planning to do a dangerous action')
462 parser.add_option('-v', '--verbose', action='store_const',
463 const=logging.DEBUG, default=logging.INFO,
464 help='Print verbose output for debugging.')
465 options, _ = parser.parse_args()
466
467 if options.dry_run or options.force or options.clean_only:
468 options.verbose = logging.DEBUG
469 logging.basicConfig(format='%(message)s', level=options.verbose)
470
471 # Work from the root directory of the checkout.
472 script_dir = os.path.dirname(os.path.abspath(__file__))
473 os.chdir(script_dir)
474
475 if sys.platform.startswith('win'):
476 def is_admin():
477 try:
478 return os.getuid() == 0
479 except AttributeError:
480 return ctypes.windll.shell32.IsUserAnAdmin() != 0
481 if not is_admin():
482 logging.error('On Windows, you now need to have administrator '
483 'privileges for the shell running %s (or '
484 '`gclient sync|runhooks`).\nPlease start another command '
Henrik Kjellander57e5fd22015-05-25 12:55:39 +0200485 'prompt as Administrator and try again.', sys.argv[0])
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000486 return 1
487
488 if not os.path.exists(CHROMIUM_CHECKOUT):
489 logging.error('Cannot find a Chromium checkout at %s. Did you run "gclient '
490 'sync" before running this script?', CHROMIUM_CHECKOUT)
491 return 2
492
493 links_database = _initialize_database(LINKS_DB)
494 try:
495 symlink_creator = WebRTCLinkSetup(links_database, options.force,
496 options.dry_run, options.prompt)
497 symlink_creator.CleanupLinks()
498 if not options.clean_only:
499 symlink_creator.CreateLinks(on_bot)
500 except LinkError as e:
501 print >> sys.stderr, e.message
502 return 3
503 finally:
504 links_database.close()
505 return 0
506
507
508if __name__ == '__main__':
509 sys.exit(main())