kjellander@webrtc.org | 8925662 | 2014-08-20 12:10:11 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. |
| 3 | # |
| 4 | # Use of this source code is governed by a BSD-style license |
| 5 | # that can be found in the LICENSE file in the root of the source |
| 6 | # tree. An additional intellectual property rights grant can be found |
| 7 | # in the file PATENTS. All contributing project authors may |
| 8 | # be found in the AUTHORS file in the root of the source tree. |
| 9 | |
kjellander@webrtc.org | 8539bd0 | 2014-10-23 12:17:58 +0000 | [diff] [blame] | 10 | """Script to download a Chromium checkout into the workspace. |
| 11 | |
| 12 | The script downloads a full Chromium Git clone and its DEPS. |
| 13 | |
| 14 | The following environment variable can be used to alter the behavior: |
| 15 | * CHROMIUM_NO_HISTORY - If set to 1, a Git checkout with no history will be |
| 16 | downloaded. This is consumes less bandwidth and disk space but is known to be |
| 17 | slower in general if you have a high-speed connection. |
| 18 | |
| 19 | After a successful sync has completed, a .last_sync_chromium file is written to |
| 20 | the chromium directory. While it exists, no more gclient sync operations will be |
| 21 | performed until the --target-revision changes or the SCRIPT_VERSION constant is |
| 22 | incremented. The file can be removed manually to force a new sync. |
| 23 | """ |
| 24 | |
kjellander@webrtc.org | 8925662 | 2014-08-20 12:10:11 +0000 | [diff] [blame] | 25 | import argparse |
| 26 | import os |
| 27 | import subprocess |
| 28 | import sys |
kjellander@webrtc.org | 7430433 | 2015-03-05 14:38:09 +0000 | [diff] [blame] | 29 | import textwrap |
kjellander@webrtc.org | 8925662 | 2014-08-20 12:10:11 +0000 | [diff] [blame] | 30 | |
iannucci@chromium.org | 1613638 | 2014-08-21 15:48:23 +0000 | [diff] [blame] | 31 | # Bump this whenever the algorithm changes and you need bots/devs to re-sync, |
| 32 | # ignoring the .last_sync_chromium file |
Henrik Kjellander | 10ba3ee | 2015-04-29 14:47:53 +0200 | [diff] [blame] | 33 | SCRIPT_VERSION = 5 |
kjellander@webrtc.org | 8925662 | 2014-08-20 12:10:11 +0000 | [diff] [blame] | 34 | |
| 35 | ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) |
kjellander@webrtc.org | 8539bd0 | 2014-10-23 12:17:58 +0000 | [diff] [blame] | 36 | CHROMIUM_NO_HISTORY = 'CHROMIUM_NO_HISTORY' |
| 37 | |
Henrik Kjellander | ae33134 | 2015-04-28 16:08:23 +0200 | [diff] [blame] | 38 | # Duplicated from depot_tools/gclient.py since we cannot depend on that: |
| 39 | DEPS_OS_CHOICES = { |
| 40 | "win32": "win", |
| 41 | "win": "win", |
| 42 | "cygwin": "win", |
| 43 | "darwin": "mac", |
| 44 | "mac": "mac", |
| 45 | "unix": "unix", |
| 46 | "linux": "unix", |
| 47 | "linux2": "unix", |
| 48 | "linux3": "unix", |
| 49 | "android": "android", |
| 50 | } |
| 51 | |
kjellander@webrtc.org | 8539bd0 | 2014-10-23 12:17:58 +0000 | [diff] [blame] | 52 | def _parse_gclient_dict(): |
| 53 | gclient_dict = {} |
| 54 | try: |
| 55 | main_gclient = os.path.join(os.path.dirname(ROOT_DIR), '.gclient') |
| 56 | with open(main_gclient, 'rb') as deps_content: |
| 57 | exec(deps_content, gclient_dict) |
| 58 | except Exception as e: |
| 59 | print >> sys.stderr, 'error while parsing .gclient:', e |
| 60 | return gclient_dict |
| 61 | |
| 62 | |
| 63 | def get_cache_dir(): |
| 64 | return _parse_gclient_dict().get('cache_dir') |
kjellander@webrtc.org | 8925662 | 2014-08-20 12:10:11 +0000 | [diff] [blame] | 65 | |
| 66 | |
| 67 | def get_target_os_list(): |
Henrik Kjellander | ae33134 | 2015-04-28 16:08:23 +0200 | [diff] [blame] | 68 | # Always add the currently running OS since the --deps option will override |
| 69 | # that if specified: |
| 70 | target_os_list = [DEPS_OS_CHOICES.get(sys.platform, 'unix')] |
| 71 | # Add any target_os entries from .gclient. |
| 72 | target_os_list += _parse_gclient_dict().get('target_os', []) |
| 73 | return ','.join(target_os_list) |
kjellander@webrtc.org | 8925662 | 2014-08-20 12:10:11 +0000 | [diff] [blame] | 74 | |
| 75 | |
| 76 | def main(): |
| 77 | CR_DIR = os.path.join(ROOT_DIR, 'chromium') |
| 78 | |
| 79 | p = argparse.ArgumentParser() |
| 80 | p.add_argument('--target-revision', required=True, |
| 81 | help='The target chromium git revision [REQUIRED]') |
| 82 | p.add_argument('--chromium-dir', default=CR_DIR, |
| 83 | help=('The path to the chromium directory to sync ' |
| 84 | '(default: %(default)r)')) |
| 85 | opts = p.parse_args() |
| 86 | opts.chromium_dir = os.path.abspath(opts.chromium_dir) |
| 87 | |
iannucci@chromium.org | 1613638 | 2014-08-21 15:48:23 +0000 | [diff] [blame] | 88 | target_os_list = get_target_os_list() |
| 89 | |
kjellander@webrtc.org | 8925662 | 2014-08-20 12:10:11 +0000 | [diff] [blame] | 90 | # Do a quick check to see if we were successful last time to make runhooks |
| 91 | # sooper fast. |
| 92 | flag_file = os.path.join(opts.chromium_dir, '.last_sync_chromium') |
iannucci@chromium.org | 1613638 | 2014-08-21 15:48:23 +0000 | [diff] [blame] | 93 | flag_file_content = '\n'.join([ |
| 94 | str(SCRIPT_VERSION), |
| 95 | opts.target_revision, |
| 96 | repr(target_os_list), |
| 97 | ]) |
kjellander@webrtc.org | dde19a6 | 2014-11-24 10:08:03 +0000 | [diff] [blame] | 98 | if (os.path.exists(os.path.join(opts.chromium_dir, 'src')) and |
| 99 | os.path.exists(flag_file)): |
kjellander@webrtc.org | 8925662 | 2014-08-20 12:10:11 +0000 | [diff] [blame] | 100 | with open(flag_file, 'r') as f: |
iannucci@chromium.org | 1613638 | 2014-08-21 15:48:23 +0000 | [diff] [blame] | 101 | if f.read() == flag_file_content: |
kjellander@webrtc.org | 8539bd0 | 2014-10-23 12:17:58 +0000 | [diff] [blame] | 102 | print 'Chromium already up to date: ', opts.target_revision |
kjellander@webrtc.org | 8925662 | 2014-08-20 12:10:11 +0000 | [diff] [blame] | 103 | return 0 |
| 104 | os.unlink(flag_file) |
| 105 | |
| 106 | env = os.environ.copy() |
kjellander@webrtc.org | 7d4e6d0 | 2014-11-27 10:41:04 +0000 | [diff] [blame] | 107 | |
| 108 | # Avoid downloading NaCl toolchain as part of the Chromium hooks. |
| 109 | env.setdefault('GYP_DEFINES', '') |
| 110 | env['GYP_DEFINES'] += ' disable_nacl=1' |
kjellander@webrtc.org | 8925662 | 2014-08-20 12:10:11 +0000 | [diff] [blame] | 111 | env['GYP_CHROMIUM_NO_ACTION'] = '1' |
| 112 | gclient_cmd = 'gclient.bat' if sys.platform.startswith('win') else 'gclient' |
| 113 | args = [ |
iannucci@chromium.org | 98d92d6 | 2014-08-20 23:53:59 +0000 | [diff] [blame] | 114 | gclient_cmd, 'sync', '--force', '--revision', 'src@'+opts.target_revision |
| 115 | ] |
| 116 | |
kjellander@webrtc.org | 3aa837c | 2014-08-20 14:25:43 +0000 | [diff] [blame] | 117 | if os.environ.get('CHROME_HEADLESS') == '1': |
kjellander@webrtc.org | 8539bd0 | 2014-10-23 12:17:58 +0000 | [diff] [blame] | 118 | # Running on a buildbot. |
kjellander@webrtc.org | 3aa837c | 2014-08-20 14:25:43 +0000 | [diff] [blame] | 119 | args.append('-vvv') |
| 120 | |
iannucci@chromium.org | 98d92d6 | 2014-08-20 23:53:59 +0000 | [diff] [blame] | 121 | if sys.platform.startswith('win'): |
iannucci@chromium.org | 286210d | 2014-08-21 02:14:11 +0000 | [diff] [blame] | 122 | cache_path = os.path.join(os.path.splitdrive(ROOT_DIR)[0] + os.path.sep, |
iannucci@chromium.org | 98d92d6 | 2014-08-20 23:53:59 +0000 | [diff] [blame] | 123 | 'b', 'git-cache') |
| 124 | else: |
| 125 | cache_path = '/b/git-cache' |
kjellander@webrtc.org | 8539bd0 | 2014-10-23 12:17:58 +0000 | [diff] [blame] | 126 | else: |
kjellander@webrtc.org | 7430433 | 2015-03-05 14:38:09 +0000 | [diff] [blame] | 127 | # Verbose, but not as verbose as on the buildbots. |
| 128 | args.append('-v') |
| 129 | |
kjellander@webrtc.org | 8539bd0 | 2014-10-23 12:17:58 +0000 | [diff] [blame] | 130 | # Support developers setting the cache_dir in .gclient. |
| 131 | cache_path = get_cache_dir() |
iannucci@chromium.org | 98d92d6 | 2014-08-20 23:53:59 +0000 | [diff] [blame] | 132 | |
kjellander@webrtc.org | 8539bd0 | 2014-10-23 12:17:58 +0000 | [diff] [blame] | 133 | # Allow for users with poor internet connections to download a Git clone |
| 134 | # without history (saves several gigs but is generally slower and doesn't work |
| 135 | # with the Git cache). |
| 136 | if os.environ.get(CHROMIUM_NO_HISTORY) == '1': |
| 137 | if cache_path: |
| 138 | print >> sys.stderr, ( |
| 139 | 'You cannot use "no-history" mode for syncing Chrome (i.e. set the ' |
| 140 | '%s environment variable to 1) when you have cache_dir configured in ' |
| 141 | 'your .gclient.' % CHROMIUM_NO_HISTORY) |
| 142 | return 1 |
| 143 | args.append('--no-history') |
| 144 | gclient_entries_file = os.path.join(opts.chromium_dir, '.gclient_entries') |
| 145 | else: |
| 146 | # Write a temporary .gclient file that has the cache_dir variable added. |
iannucci@chromium.org | 286210d | 2014-08-21 02:14:11 +0000 | [diff] [blame] | 147 | gclientfile = os.path.join(opts.chromium_dir, '.gclient') |
| 148 | with open(gclientfile, 'rb') as spec: |
iannucci@chromium.org | 98d92d6 | 2014-08-20 23:53:59 +0000 | [diff] [blame] | 149 | spec = spec.read().splitlines() |
| 150 | spec[-1] = 'cache_dir = %r' % (cache_path,) |
kjellander@webrtc.org | 8539bd0 | 2014-10-23 12:17:58 +0000 | [diff] [blame] | 151 | with open(gclientfile + '.tmp', 'wb') as f: |
iannucci@chromium.org | 286210d | 2014-08-21 02:14:11 +0000 | [diff] [blame] | 152 | f.write('\n'.join(spec)) |
| 153 | |
| 154 | args += [ |
kjellander@webrtc.org | 8539bd0 | 2014-10-23 12:17:58 +0000 | [diff] [blame] | 155 | '--gclientfile', '.gclient.tmp', |
iannucci@chromium.org | 286210d | 2014-08-21 02:14:11 +0000 | [diff] [blame] | 156 | '--delete_unversioned_trees', '--reset', '--upstream' |
| 157 | ] |
kjellander@webrtc.org | 6c6680a | 2014-09-27 18:41:03 +0000 | [diff] [blame] | 158 | gclient_entries_file = os.path.join(opts.chromium_dir, |
kjellander@webrtc.org | 8539bd0 | 2014-10-23 12:17:58 +0000 | [diff] [blame] | 159 | '.gclient.tmp_entries') |
kjellander@webrtc.org | 6c6680a | 2014-09-27 18:41:03 +0000 | [diff] [blame] | 160 | |
| 161 | # To avoid gclient sync problems when DEPS entries have been removed we must |
| 162 | # wipe the gclient's entries file that contains cached URLs for all DEPS. |
| 163 | if os.path.exists(gclient_entries_file): |
| 164 | os.unlink(gclient_entries_file) |
iannucci@chromium.org | 98d92d6 | 2014-08-20 23:53:59 +0000 | [diff] [blame] | 165 | |
kjellander@webrtc.org | 8925662 | 2014-08-20 12:10:11 +0000 | [diff] [blame] | 166 | if target_os_list: |
| 167 | args += ['--deps=' + target_os_list] |
| 168 | |
kjellander@webrtc.org | 7430433 | 2015-03-05 14:38:09 +0000 | [diff] [blame] | 169 | print textwrap.dedent("""\ |
Henrik Kjellander | b3fc48b | 2015-03-27 14:25:35 +0100 | [diff] [blame] | 170 | +---------------------------------------------------------------------+ |
| 171 | | NOTICE: This sync of Chromium will take a long time as several | |
| 172 | | gigabytes of data are downloaded. If this is your initial | |
| 173 | | sync and it's interrupted, try running 'gclient sync' again.| |
| 174 | | If that fails, wipe everything clean and start over again. | |
| 175 | +---------------------------------------------------------------------+""") |
kjellander@webrtc.org | 8925662 | 2014-08-20 12:10:11 +0000 | [diff] [blame] | 176 | print 'Running "%s" in %s' % (' '.join(args), opts.chromium_dir) |
| 177 | ret = subprocess.call(args, cwd=opts.chromium_dir, env=env) |
| 178 | if ret == 0: |
| 179 | with open(flag_file, 'wb') as f: |
iannucci@chromium.org | 1613638 | 2014-08-21 15:48:23 +0000 | [diff] [blame] | 180 | f.write(flag_file_content) |
kjellander@webrtc.org | 8925662 | 2014-08-20 12:10:11 +0000 | [diff] [blame] | 181 | |
| 182 | return ret |
| 183 | |
| 184 | |
| 185 | if __name__ == '__main__': |
| 186 | sys.exit(main()) |