blob: 9aeb1e876405c2309fea90933f23b28c9c73c04f [file] [log] [blame]
kjellander@webrtc.org89256622014-08-20 12:10:11 +00001#!/usr/bin/env python
2# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS. All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
9
10"""Setup links to a Chromium checkout for WebRTC.
11
12WebRTC standalone shares a lot of dependencies and build tools with Chromium.
13To do this, many of the paths of a Chromium checkout is emulated by creating
14symlinks to files and directories. This script handles the setup of symlinks to
15achieve this.
16
17It also handles cleanup of the legacy Subversion-based approach that was used
18before Chrome switched over their master repo from Subversion to Git.
19"""
20
21
22import ctypes
23import errno
24import logging
25import optparse
26import os
27import shelve
28import shutil
29import subprocess
30import sys
31import textwrap
32
33
34DIRECTORIES = [
35 'build',
36 'buildtools',
37 'google_apis', # Needed by build/common.gypi.
38 'net',
39 'testing',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000040 'third_party/binutils',
41 'third_party/boringssl',
42 'third_party/colorama',
43 'third_party/drmemory',
44 'third_party/expat',
45 'third_party/icu',
kjellander@webrtc.org4e4fe4f2014-10-01 08:03:19 +000046 'third_party/instrumented_libraries',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000047 'third_party/jsoncpp',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000048 'third_party/libjpeg',
49 'third_party/libjpeg_turbo',
50 'third_party/libsrtp',
kjellander@webrtc.org7d4e6d02014-11-27 10:41:04 +000051 'third_party/libudev',
kjellanderd6024e32015-09-28 21:16:48 -070052 'third_party/libvpx_new',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000053 'third_party/libyuv',
54 'third_party/llvm-build',
Patrik Höglundc92c23d2015-08-31 11:30:14 +020055 'third_party/lss',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000056 'third_party/nss',
tkchin@webrtc.org3a63a3c2015-01-06 07:21:34 +000057 'third_party/ocmock',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000058 'third_party/openmax_dl',
59 'third_party/opus',
Patrik Höglundc92c23d2015-08-31 11:30:14 +020060 'third_party/proguard',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000061 'third_party/protobuf',
62 'third_party/sqlite',
63 'third_party/syzygy',
64 'third_party/usrsctp',
65 'third_party/yasm',
kjellander@webrtc.org3bd41562014-09-01 11:06:37 +000066 'third_party/zlib',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000067 'tools/clang',
68 'tools/generate_library_loader',
69 'tools/gn',
70 'tools/gyp',
71 'tools/memory',
72 'tools/protoc_wrapper',
73 'tools/python',
74 'tools/swarming_client',
75 'tools/valgrind',
Andrew MacDonald65de7d22015-05-19 11:37:34 -070076 'tools/vim',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000077 'tools/win',
Henrik Kjellander9589e2a2015-10-22 06:48:21 +020078 'tools/xdisplaycheck',
kjellander@webrtc.org89256622014-08-20 12:10:11 +000079]
80
kjellander@webrtc.org3bd41562014-09-01 11:06:37 +000081from sync_chromium import get_target_os_list
Henrik Kjellanderca843022015-06-09 10:51:22 +020082target_os = get_target_os_list()
83if 'android' in target_os:
kjellander@webrtc.org3bd41562014-09-01 11:06:37 +000084 DIRECTORIES += [
85 'base',
Henrik Kjellander94a12322015-06-09 14:56:20 +020086 'third_party/android_platform',
kjellander@webrtc.org3bd41562014-09-01 11:06:37 +000087 'third_party/android_testrunner',
88 'third_party/android_tools',
kjellander@webrtc.orgcbe7ca82015-01-06 07:24:27 +000089 'third_party/appurify-python',
kjellander@webrtc.orgb8caf6a2014-09-30 18:05:02 +000090 'third_party/ashmem',
Henrik Kjellandereecbab72015-09-16 19:19:04 +020091 'third_party/ijar',
kjellander@webrtc.orgb8caf6a2014-09-30 18:05:02 +000092 'third_party/jsr-305',
Henrik Kjellander10ba3ee2015-04-29 14:47:53 +020093 'third_party/junit',
kjellander@webrtc.orgb8caf6a2014-09-30 18:05:02 +000094 'third_party/libevent',
95 'third_party/libxml',
Henrik Kjellander10ba3ee2015-04-29 14:47:53 +020096 'third_party/mockito',
kjellander@webrtc.orgb8caf6a2014-09-30 18:05:02 +000097 'third_party/modp_b64',
kjellander@webrtc.orgcbe7ca82015-01-06 07:24:27 +000098 'third_party/requests',
Henrik Kjellander10ba3ee2015-04-29 14:47:53 +020099 'third_party/robolectric',
kjellander@webrtc.org3bd41562014-09-01 11:06:37 +0000100 'tools/android',
kjellander@webrtc.orgbfdee692015-02-03 15:23:34 +0000101 'tools/grit',
marpan@webrtc.org4765ca52014-11-03 20:10:26 +0000102 'tools/relocation_packer'
kjellander@webrtc.org3bd41562014-09-01 11:06:37 +0000103 ]
Henrik Kjellanderca843022015-06-09 10:51:22 +0200104if 'ios' in target_os:
105 DIRECTORIES.append('third_party/class-dump')
kjellander@webrtc.org3bd41562014-09-01 11:06:37 +0000106
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000107FILES = {
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000108 'tools/find_depot_tools.py': None,
Henrik Kjellanderd6d27e72015-09-25 22:19:11 +0200109 'tools/isolate_driver.py': None,
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000110 'third_party/BUILD.gn': None,
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000111}
112
kjellander@webrtc.orge94f83a2014-09-18 13:47:23 +0000113ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000114CHROMIUM_CHECKOUT = os.path.join('chromium', 'src')
115LINKS_DB = 'links'
116
117# Version management to make future upgrades/downgrades easier to support.
118SCHEMA_VERSION = 1
119
120
121def query_yes_no(question, default=False):
122 """Ask a yes/no question via raw_input() and return their answer.
123
124 Modified from http://stackoverflow.com/a/3041990.
125 """
126 prompt = " [%s/%%s]: "
127 prompt = prompt % ('Y' if default is True else 'y')
128 prompt = prompt % ('N' if default is False else 'n')
129
130 if default is None:
131 default = 'INVALID'
132
133 while True:
134 sys.stdout.write(question + prompt)
135 choice = raw_input().lower()
136 if choice == '' and default != 'INVALID':
137 return default
138
139 if 'yes'.startswith(choice):
140 return True
141 elif 'no'.startswith(choice):
142 return False
143
144 print "Please respond with 'yes' or 'no' (or 'y' or 'n')."
145
146
147# Actions
148class Action(object):
149 def __init__(self, dangerous):
150 self.dangerous = dangerous
151
152 def announce(self, planning):
153 """Log a description of this action.
154
155 Args:
156 planning - True iff we're in the planning stage, False if we're in the
157 doit stage.
158 """
159 pass
160
161 def doit(self, links_db):
162 """Execute the action, recording what we did to links_db, if necessary."""
163 pass
164
165
166class Remove(Action):
167 def __init__(self, path, dangerous):
168 super(Remove, self).__init__(dangerous)
169 self._priority = 0
170 self._path = path
171
172 def announce(self, planning):
173 log = logging.warn
174 filesystem_type = 'file'
175 if not self.dangerous:
176 log = logging.info
177 filesystem_type = 'link'
178 if planning:
179 log('Planning to remove %s: %s', filesystem_type, self._path)
180 else:
181 log('Removing %s: %s', filesystem_type, self._path)
182
Henrik Kjellander57e5fd22015-05-25 12:55:39 +0200183 def doit(self, _):
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000184 os.remove(self._path)
185
186
187class Rmtree(Action):
188 def __init__(self, path):
189 super(Rmtree, self).__init__(dangerous=True)
190 self._priority = 0
191 self._path = path
192
193 def announce(self, planning):
194 if planning:
195 logging.warn('Planning to remove directory: %s', self._path)
196 else:
197 logging.warn('Removing directory: %s', self._path)
198
Henrik Kjellander57e5fd22015-05-25 12:55:39 +0200199 def doit(self, _):
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000200 if sys.platform.startswith('win'):
201 # shutil.rmtree() doesn't work on Windows if any of the directories are
202 # read-only, which svn repositories are.
203 subprocess.check_call(['rd', '/q', '/s', self._path], shell=True)
204 else:
205 shutil.rmtree(self._path)
206
207
208class Makedirs(Action):
209 def __init__(self, path):
210 super(Makedirs, self).__init__(dangerous=False)
211 self._priority = 1
212 self._path = path
213
Henrik Kjellander57e5fd22015-05-25 12:55:39 +0200214 def doit(self, _):
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000215 try:
216 os.makedirs(self._path)
217 except OSError as e:
218 if e.errno != errno.EEXIST:
219 raise
220
221
222class Symlink(Action):
223 def __init__(self, source_path, link_path):
224 super(Symlink, self).__init__(dangerous=False)
225 self._priority = 2
226 self._source_path = source_path
227 self._link_path = link_path
228
229 def announce(self, planning):
230 if planning:
231 logging.info(
232 'Planning to create link from %s to %s', self._link_path,
233 self._source_path)
234 else:
235 logging.debug(
236 'Linking from %s to %s', self._link_path, self._source_path)
237
238 def doit(self, links_db):
239 # Files not in the root directory need relative path calculation.
240 # On Windows, use absolute paths instead since NTFS doesn't seem to support
241 # relative paths for symlinks.
242 if sys.platform.startswith('win'):
243 source_path = os.path.abspath(self._source_path)
244 else:
245 if os.path.dirname(self._link_path) != self._link_path:
246 source_path = os.path.relpath(self._source_path,
247 os.path.dirname(self._link_path))
248
249 os.symlink(source_path, os.path.abspath(self._link_path))
250 links_db[self._source_path] = self._link_path
251
252
253class LinkError(IOError):
254 """Failed to create a link."""
255 pass
256
257
258# Handles symlink creation on the different platforms.
259if sys.platform.startswith('win'):
260 def symlink(source_path, link_path):
261 flag = 1 if os.path.isdir(source_path) else 0
262 if not ctypes.windll.kernel32.CreateSymbolicLinkW(
263 unicode(link_path), unicode(source_path), flag):
264 raise OSError('Failed to create symlink to %s. Notice that only NTFS '
265 'version 5.0 and up has all the needed APIs for '
266 'creating symlinks.' % source_path)
267 os.symlink = symlink
268
269
Henrik Kjellander57e5fd22015-05-25 12:55:39 +0200270class WebRTCLinkSetup(object):
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000271 def __init__(self, links_db, force=False, dry_run=False, prompt=False):
272 self._force = force
273 self._dry_run = dry_run
274 self._prompt = prompt
275 self._links_db = links_db
276
277 def CreateLinks(self, on_bot):
278 logging.debug('CreateLinks')
279 # First, make a plan of action
280 actions = []
281
282 for source_path, link_path in FILES.iteritems():
283 actions += self._ActionForPath(
284 source_path, link_path, check_fn=os.path.isfile, check_msg='files')
285 for source_dir in DIRECTORIES:
286 actions += self._ActionForPath(
287 source_dir, None, check_fn=os.path.isdir,
288 check_msg='directories')
289
kjellander@webrtc.orge94f83a2014-09-18 13:47:23 +0000290 if not on_bot and self._force:
291 # When making the manual switch from legacy SVN checkouts to the new
292 # Git-based Chromium DEPS, the .gclient_entries file that contains cached
293 # URLs for all DEPS entries must be removed to avoid future sync problems.
294 entries_file = os.path.join(os.path.dirname(ROOT_DIR), '.gclient_entries')
295 if os.path.exists(entries_file):
296 actions.append(Remove(entries_file, dangerous=True))
297
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000298 actions.sort()
299
300 if self._dry_run:
301 for action in actions:
302 action.announce(planning=True)
303 logging.info('Not doing anything because dry-run was specified.')
304 sys.exit(0)
305
306 if any(a.dangerous for a in actions):
307 logging.warn('Dangerous actions:')
308 for action in (a for a in actions if a.dangerous):
309 action.announce(planning=True)
310 print
311
312 if not self._force:
313 logging.error(textwrap.dedent("""\
314 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
315 A C T I O N R E Q I R E D
316 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
317
318 Because chromium/src is transitioning to Git (from SVN), we needed to
319 change the way that the WebRTC standalone checkout works. Instead of
320 individually syncing subdirectories of Chromium in SVN, we're now
321 syncing Chromium (and all of its DEPS, as defined by its own DEPS file),
322 into the `chromium/src` directory.
323
324 As such, all Chromium directories which are currently pulled by DEPS are
325 now replaced with a symlink into the full Chromium checkout.
326
327 To avoid disrupting developers, we've chosen to not delete your
328 directories forcibly, in case you have some work in progress in one of
329 them :).
330
331 ACTION REQUIRED:
332 Before running `gclient sync|runhooks` again, you must run:
333 %s%s --force
334
335 Which will replace all directories which now must be symlinks, after
336 prompting with a summary of the work-to-be-done.
337 """), 'python ' if sys.platform.startswith('win') else '', sys.argv[0])
338 sys.exit(1)
339 elif self._prompt:
340 if not query_yes_no('Would you like to perform the above plan?'):
341 sys.exit(1)
342
343 for action in actions:
344 action.announce(planning=False)
345 action.doit(self._links_db)
346
347 if not on_bot and self._force:
348 logging.info('Completed!\n\nNow run `gclient sync|runhooks` again to '
349 'let the remaining hooks (that probably were interrupted) '
350 'execute.')
351
352 def CleanupLinks(self):
353 logging.debug('CleanupLinks')
354 for source, link_path in self._links_db.iteritems():
355 if source == 'SCHEMA_VERSION':
356 continue
357 if os.path.islink(link_path) or sys.platform.startswith('win'):
358 # os.path.islink() always returns false on Windows
359 # See http://bugs.python.org/issue13143.
360 logging.debug('Removing link to %s at %s', source, link_path)
361 if not self._dry_run:
362 if os.path.exists(link_path):
363 if sys.platform.startswith('win') and os.path.isdir(link_path):
Henrik Kjellanderc444de62015-04-29 11:27:22 +0200364 subprocess.check_call(['rmdir', '/q', '/s', link_path],
365 shell=True)
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000366 else:
367 os.remove(link_path)
368 del self._links_db[source]
369
370 @staticmethod
371 def _ActionForPath(source_path, link_path=None, check_fn=None,
372 check_msg=None):
373 """Create zero or more Actions to link to a file or directory.
374
375 This will be a symlink on POSIX platforms. On Windows this requires
376 that NTFS is version 5.0 or higher (Vista or newer).
377
378 Args:
379 source_path: Path relative to the Chromium checkout root.
380 For readability, the path may contain slashes, which will
381 automatically be converted to the right path delimiter on Windows.
382 link_path: The location for the link to create. If omitted it will be the
383 same path as source_path.
384 check_fn: A function returning true if the type of filesystem object is
385 correct for the attempted call. Otherwise an error message with
386 check_msg will be printed.
387 check_msg: String used to inform the user of an invalid attempt to create
388 a file.
389 Returns:
390 A list of Action objects.
391 """
392 def fix_separators(path):
393 if sys.platform.startswith('win'):
394 return path.replace(os.altsep, os.sep)
395 else:
396 return path
397
398 assert check_fn
399 assert check_msg
400 link_path = link_path or source_path
401 link_path = fix_separators(link_path)
402
403 source_path = fix_separators(source_path)
404 source_path = os.path.join(CHROMIUM_CHECKOUT, source_path)
405 if os.path.exists(source_path) and not check_fn:
406 raise LinkError('_LinkChromiumPath can only be used to link to %s: '
407 'Tried to link to: %s' % (check_msg, source_path))
408
409 if not os.path.exists(source_path):
410 logging.debug('Silently ignoring missing source: %s. This is to avoid '
411 'errors on platform-specific dependencies.', source_path)
412 return []
413
414 actions = []
415
416 if os.path.exists(link_path) or os.path.islink(link_path):
417 if os.path.islink(link_path):
418 actions.append(Remove(link_path, dangerous=False))
419 elif os.path.isfile(link_path):
420 actions.append(Remove(link_path, dangerous=True))
421 elif os.path.isdir(link_path):
422 actions.append(Rmtree(link_path))
423 else:
424 raise LinkError('Don\'t know how to plan: %s' % link_path)
425
426 # Create parent directories to the target link if needed.
427 target_parent_dirs = os.path.dirname(link_path)
428 if (target_parent_dirs and
429 target_parent_dirs != link_path and
430 not os.path.exists(target_parent_dirs)):
431 actions.append(Makedirs(target_parent_dirs))
432
433 actions.append(Symlink(source_path, link_path))
434
435 return actions
436
437def _initialize_database(filename):
438 links_database = shelve.open(filename)
439
440 # Wipe the database if this version of the script ends up looking at a
441 # newer (future) version of the links db, just to be sure.
442 version = links_database.get('SCHEMA_VERSION')
443 if version and version != SCHEMA_VERSION:
444 logging.info('Found database with schema version %s while this script only '
445 'supports %s. Wiping previous database contents.', version,
446 SCHEMA_VERSION)
447 links_database.clear()
448 links_database['SCHEMA_VERSION'] = SCHEMA_VERSION
449 return links_database
450
451
452def main():
453 on_bot = os.environ.get('CHROME_HEADLESS') == '1'
454
455 parser = optparse.OptionParser()
456 parser.add_option('-d', '--dry-run', action='store_true', default=False,
457 help='Print what would be done, but don\'t perform any '
458 'operations. This will automatically set logging to '
459 'verbose.')
460 parser.add_option('-c', '--clean-only', action='store_true', default=False,
461 help='Only clean previously created links, don\'t create '
462 'new ones. This will automatically set logging to '
463 'verbose.')
464 parser.add_option('-f', '--force', action='store_true', default=on_bot,
465 help='Force link creation. CAUTION: This deletes existing '
466 'folders and files in the locations where links are '
467 'about to be created.')
468 parser.add_option('-n', '--no-prompt', action='store_false', dest='prompt',
469 default=(not on_bot),
470 help='Prompt if we\'re planning to do a dangerous action')
471 parser.add_option('-v', '--verbose', action='store_const',
472 const=logging.DEBUG, default=logging.INFO,
473 help='Print verbose output for debugging.')
474 options, _ = parser.parse_args()
475
476 if options.dry_run or options.force or options.clean_only:
477 options.verbose = logging.DEBUG
478 logging.basicConfig(format='%(message)s', level=options.verbose)
479
480 # Work from the root directory of the checkout.
481 script_dir = os.path.dirname(os.path.abspath(__file__))
482 os.chdir(script_dir)
483
484 if sys.platform.startswith('win'):
485 def is_admin():
486 try:
487 return os.getuid() == 0
488 except AttributeError:
489 return ctypes.windll.shell32.IsUserAnAdmin() != 0
490 if not is_admin():
491 logging.error('On Windows, you now need to have administrator '
492 'privileges for the shell running %s (or '
493 '`gclient sync|runhooks`).\nPlease start another command '
Henrik Kjellander57e5fd22015-05-25 12:55:39 +0200494 'prompt as Administrator and try again.', sys.argv[0])
kjellander@webrtc.org89256622014-08-20 12:10:11 +0000495 return 1
496
497 if not os.path.exists(CHROMIUM_CHECKOUT):
498 logging.error('Cannot find a Chromium checkout at %s. Did you run "gclient '
499 'sync" before running this script?', CHROMIUM_CHECKOUT)
500 return 2
501
502 links_database = _initialize_database(LINKS_DB)
503 try:
504 symlink_creator = WebRTCLinkSetup(links_database, options.force,
505 options.dry_run, options.prompt)
506 symlink_creator.CleanupLinks()
507 if not options.clean_only:
508 symlink_creator.CreateLinks(on_bot)
509 except LinkError as e:
510 print >> sys.stderr, e.message
511 return 3
512 finally:
513 links_database.close()
514 return 0
515
516
517if __name__ == '__main__':
518 sys.exit(main())