Hal Canary | a6f6f77 | 2017-01-27 11:57:13 -0500 | [diff] [blame] | 1 | #!/usr/bin/env python |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 2 | # Copyright 2014 Google Inc. |
| 3 | # |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | |
| 7 | |
| 8 | """Parse a DEPS file and git checkout all of the dependencies. |
| 9 | |
| 10 | Args: |
| 11 | An optional list of deps_os values. |
| 12 | |
| 13 | Environment Variables: |
Ben Wagner | 96aa535 | 2018-06-19 10:34:32 -0400 | [diff] [blame^] | 14 | GIT_EXECUTABLE: path to "git" binary; if unset, will look for git in |
| 15 | your default path. |
halcanary | 20fb7c6 | 2014-06-25 13:28:29 -0700 | [diff] [blame] | 16 | |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 17 | GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset, |
| 18 | will use the file ../DEPS relative to this script's directory. |
| 19 | |
| 20 | GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages. |
| 21 | |
| 22 | Git Config: |
| 23 | To disable syncing of a single repository: |
| 24 | cd path/to/repository |
| 25 | git config sync-deps.disable true |
| 26 | |
| 27 | To re-enable sync: |
| 28 | cd path/to/repository |
| 29 | git config --unset sync-deps.disable |
| 30 | """ |
| 31 | |
| 32 | |
| 33 | import os |
| 34 | import subprocess |
| 35 | import sys |
| 36 | import threading |
| 37 | |
halcanary | 20fb7c6 | 2014-06-25 13:28:29 -0700 | [diff] [blame] | 38 | |
| 39 | def git_executable(): |
| 40 | """Find the git executable. |
| 41 | |
| 42 | Returns: |
| 43 | A string suitable for passing to subprocess functions, or None. |
| 44 | """ |
| 45 | envgit = os.environ.get('GIT_EXECUTABLE') |
Ben Wagner | 96aa535 | 2018-06-19 10:34:32 -0400 | [diff] [blame^] | 46 | searchlist = ['git'] |
halcanary | 20fb7c6 | 2014-06-25 13:28:29 -0700 | [diff] [blame] | 47 | if envgit: |
| 48 | searchlist.insert(0, envgit) |
| 49 | with open(os.devnull, 'w') as devnull: |
| 50 | for git in searchlist: |
| 51 | try: |
| 52 | subprocess.call([git, '--version'], stdout=devnull) |
| 53 | except (OSError,): |
| 54 | continue |
| 55 | return git |
| 56 | return None |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 57 | |
| 58 | |
| 59 | DEFAULT_DEPS_PATH = os.path.normpath( |
| 60 | os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS')) |
| 61 | |
| 62 | |
| 63 | def usage(deps_file_path = None): |
| 64 | sys.stderr.write( |
| 65 | 'Usage: run to grab dependencies, with optional platform support:\n') |
Hal Canary | a6f6f77 | 2017-01-27 11:57:13 -0500 | [diff] [blame] | 66 | sys.stderr.write(' %s %s' % (sys.executable, __file__)) |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 67 | if deps_file_path: |
Hal Canary | a6f6f77 | 2017-01-27 11:57:13 -0500 | [diff] [blame] | 68 | parsed_deps = parse_file_to_dict(deps_file_path) |
| 69 | if 'deps_os' in parsed_deps: |
| 70 | for deps_os in parsed_deps['deps_os']: |
| 71 | sys.stderr.write(' [%s]' % deps_os) |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 72 | sys.stderr.write('\n\n') |
| 73 | sys.stderr.write(__doc__) |
| 74 | |
| 75 | |
| 76 | def git_repository_sync_is_disabled(git, directory): |
| 77 | try: |
| 78 | disable = subprocess.check_output( |
| 79 | [git, 'config', 'sync-deps.disable'], cwd=directory) |
| 80 | return disable.lower().strip() in ['true', '1', 'yes', 'on'] |
| 81 | except subprocess.CalledProcessError: |
| 82 | return False |
| 83 | |
| 84 | |
commit-bot@chromium.org | aae6e6f | 2014-04-21 19:03:05 +0000 | [diff] [blame] | 85 | def is_git_toplevel(git, directory): |
| 86 | """Return true iff the directory is the top level of a Git repository. |
| 87 | |
| 88 | Args: |
| 89 | git (string) the git executable |
| 90 | |
| 91 | directory (string) the path into which the repository |
| 92 | is expected to be checked out. |
| 93 | """ |
| 94 | try: |
| 95 | toplevel = subprocess.check_output( |
| 96 | [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip() |
commit-bot@chromium.org | 2349a1e | 2014-04-28 13:39:47 +0000 | [diff] [blame] | 97 | return os.path.realpath(directory) == os.path.realpath(toplevel) |
commit-bot@chromium.org | aae6e6f | 2014-04-21 19:03:05 +0000 | [diff] [blame] | 98 | except subprocess.CalledProcessError: |
| 99 | return False |
| 100 | |
| 101 | |
Hal Canary | a6f6f77 | 2017-01-27 11:57:13 -0500 | [diff] [blame] | 102 | def status(directory, checkoutable): |
| 103 | def truncate(s, length): |
| 104 | return s if len(s) <= length else s[:(length - 3)] + '...' |
| 105 | dlen = 36 |
| 106 | directory = truncate(directory, dlen) |
| 107 | checkoutable = truncate(checkoutable, 40) |
| 108 | sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable)) |
| 109 | |
| 110 | |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 111 | def git_checkout_to_directory(git, repo, checkoutable, directory, verbose): |
| 112 | """Checkout (and clone if needed) a Git repository. |
| 113 | |
| 114 | Args: |
| 115 | git (string) the git executable |
| 116 | |
| 117 | repo (string) the location of the repository, suitable |
| 118 | for passing to `git clone`. |
| 119 | |
| 120 | checkoutable (string) a tag, branch, or commit, suitable for |
| 121 | passing to `git checkout` |
| 122 | |
| 123 | directory (string) the path into which the repository |
| 124 | should be checked out. |
| 125 | |
| 126 | verbose (boolean) |
| 127 | |
| 128 | Raises an exception if any calls to git fail. |
| 129 | """ |
| 130 | if not os.path.isdir(directory): |
| 131 | subprocess.check_call( |
| 132 | [git, 'clone', '--quiet', repo, directory]) |
| 133 | |
commit-bot@chromium.org | aae6e6f | 2014-04-21 19:03:05 +0000 | [diff] [blame] | 134 | if not is_git_toplevel(git, directory): |
| 135 | # if the directory exists, but isn't a git repo, you will modify |
| 136 | # the parent repostory, which isn't what you want. |
| 137 | sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory) |
| 138 | return |
| 139 | |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 140 | # Check to see if this repo is disabled. Quick return. |
| 141 | if git_repository_sync_is_disabled(git, directory): |
| 142 | sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory) |
| 143 | return |
| 144 | |
Hal Canary | 73eae98 | 2017-02-13 13:40:56 -0500 | [diff] [blame] | 145 | with open(os.devnull, 'w') as devnull: |
| 146 | # If this fails, we will fetch before trying again. Don't spam user |
| 147 | # with error infomation. |
| 148 | if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable], |
| 149 | cwd=directory, stderr=devnull): |
| 150 | # if this succeeds, skip slow `git fetch`. |
| 151 | if verbose: |
| 152 | status(directory, checkoutable) # Success. |
| 153 | return |
halcanary | 6aff54c | 2015-11-03 09:50:03 -0800 | [diff] [blame] | 154 | |
Hal Canary | a6f6f77 | 2017-01-27 11:57:13 -0500 | [diff] [blame] | 155 | # If the repo has changed, always force use of the correct repo. |
| 156 | # If origin already points to repo, this is a quick no-op. |
| 157 | subprocess.check_call( |
| 158 | [git, 'remote', 'set-url', 'origin', repo], cwd=directory) |
| 159 | |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 160 | subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory) |
| 161 | |
Hal Canary | 49deb7e | 2017-02-07 12:05:47 -0500 | [diff] [blame] | 162 | subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory) |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 163 | |
| 164 | if verbose: |
Hal Canary | a6f6f77 | 2017-01-27 11:57:13 -0500 | [diff] [blame] | 165 | status(directory, checkoutable) # Success. |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 166 | |
| 167 | |
| 168 | def parse_file_to_dict(path): |
| 169 | dictionary = {} |
| 170 | execfile(path, dictionary) |
| 171 | return dictionary |
| 172 | |
| 173 | |
halcanary | c6c0624 | 2014-08-26 12:06:47 -0700 | [diff] [blame] | 174 | def git_sync_deps(deps_file_path, command_line_os_requests, verbose): |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 175 | """Grab dependencies, with optional platform support. |
| 176 | |
| 177 | Args: |
| 178 | deps_file_path (string) Path to the DEPS file. |
| 179 | |
halcanary | c6c0624 | 2014-08-26 12:06:47 -0700 | [diff] [blame] | 180 | command_line_os_requests (list of strings) Can be empty list. |
| 181 | List of strings that should each be a key in the deps_os |
| 182 | dictionary in the DEPS file. |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 183 | |
halcanary | c6c0624 | 2014-08-26 12:06:47 -0700 | [diff] [blame] | 184 | Raises git Exceptions. |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 185 | """ |
halcanary | 20fb7c6 | 2014-06-25 13:28:29 -0700 | [diff] [blame] | 186 | git = git_executable() |
| 187 | assert git |
| 188 | |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 189 | deps_file_directory = os.path.dirname(deps_file_path) |
halcanary | c6c0624 | 2014-08-26 12:06:47 -0700 | [diff] [blame] | 190 | deps_file = parse_file_to_dict(deps_file_path) |
| 191 | dependencies = deps_file['deps'].copy() |
Hal Canary | a6f6f77 | 2017-01-27 11:57:13 -0500 | [diff] [blame] | 192 | os_specific_dependencies = deps_file.get('deps_os', dict()) |
| 193 | if 'all' in command_line_os_requests: |
| 194 | for value in os_specific_dependencies.itervalues(): |
| 195 | dependencies.update(value) |
| 196 | else: |
| 197 | for os_name in command_line_os_requests: |
| 198 | # Add OS-specific dependencies |
| 199 | if os_name in os_specific_dependencies: |
| 200 | dependencies.update(os_specific_dependencies[os_name]) |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 201 | for directory in dependencies: |
Hal Canary | a6f6f77 | 2017-01-27 11:57:13 -0500 | [diff] [blame] | 202 | for other_dir in dependencies: |
| 203 | if directory.startswith(other_dir + '/'): |
| 204 | raise Exception('%r is parent of %r' % (other_dir, directory)) |
| 205 | list_of_arg_lists = [] |
| 206 | for directory in sorted(dependencies): |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 207 | if '@' in dependencies[directory]: |
| 208 | repo, checkoutable = dependencies[directory].split('@', 1) |
| 209 | else: |
halcanary | 6aff54c | 2015-11-03 09:50:03 -0800 | [diff] [blame] | 210 | raise Exception("please specify commit or tag") |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 211 | |
| 212 | relative_directory = os.path.join(deps_file_directory, directory) |
| 213 | |
| 214 | list_of_arg_lists.append( |
halcanary | 20fb7c6 | 2014-06-25 13:28:29 -0700 | [diff] [blame] | 215 | (git, repo, checkoutable, relative_directory, verbose)) |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 216 | |
| 217 | multithread(git_checkout_to_directory, list_of_arg_lists) |
| 218 | |
halcanary | c6c0624 | 2014-08-26 12:06:47 -0700 | [diff] [blame] | 219 | for directory in deps_file.get('recursedeps', []): |
halcanary | 5164a97 | 2014-08-21 13:17:43 -0700 | [diff] [blame] | 220 | recursive_path = os.path.join(deps_file_directory, directory, 'DEPS') |
halcanary | c6c0624 | 2014-08-26 12:06:47 -0700 | [diff] [blame] | 221 | git_sync_deps(recursive_path, command_line_os_requests, verbose) |
halcanary | 5164a97 | 2014-08-21 13:17:43 -0700 | [diff] [blame] | 222 | |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 223 | |
| 224 | def multithread(function, list_of_arg_lists): |
| 225 | # for args in list_of_arg_lists: |
| 226 | # function(*args) |
| 227 | # return |
| 228 | threads = [] |
| 229 | for args in list_of_arg_lists: |
| 230 | thread = threading.Thread(None, function, None, args) |
| 231 | thread.start() |
| 232 | threads.append(thread) |
| 233 | for thread in threads: |
| 234 | thread.join() |
| 235 | |
| 236 | |
| 237 | def main(argv): |
| 238 | deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH) |
| 239 | verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False)) |
halcanary | c6c0624 | 2014-08-26 12:06:47 -0700 | [diff] [blame] | 240 | |
| 241 | if '--help' in argv or '-h' in argv: |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 242 | usage(deps_file_path) |
| 243 | return 1 |
| 244 | |
halcanary | c6c0624 | 2014-08-26 12:06:47 -0700 | [diff] [blame] | 245 | git_sync_deps(deps_file_path, argv, verbose) |
Hal Canary | a6f6f77 | 2017-01-27 11:57:13 -0500 | [diff] [blame] | 246 | subprocess.check_call( |
| 247 | [sys.executable, |
| 248 | os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')]) |
halcanary | c6c0624 | 2014-08-26 12:06:47 -0700 | [diff] [blame] | 249 | return 0 |
| 250 | |
commit-bot@chromium.org | 71e800a | 2014-04-16 19:21:00 +0000 | [diff] [blame] | 251 | |
| 252 | if __name__ == '__main__': |
| 253 | exit(main(sys.argv[1:])) |