blob: ca1ba47a7598f933d4f27cf7b2bd6a05db578f19 [file] [log] [blame]
Hal Canarya6f6f772017-01-27 11:57:13 -05001#!/usr/bin/env python
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +00002# Copyright 2014 Google Inc.
3#
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7
8"""Parse a DEPS file and git checkout all of the dependencies.
9
10Args:
11 An optional list of deps_os values.
12
13Environment Variables:
Ben Wagner96aa5352018-06-19 10:34:32 -040014 GIT_EXECUTABLE: path to "git" binary; if unset, will look for git in
15 your default path.
halcanary20fb7c62014-06-25 13:28:29 -070016
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +000017 GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset,
18 will use the file ../DEPS relative to this script's directory.
19
20 GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages.
21
22Git Config:
23 To disable syncing of a single repository:
24 cd path/to/repository
25 git config sync-deps.disable true
26
27 To re-enable sync:
28 cd path/to/repository
29 git config --unset sync-deps.disable
30"""
31
32
33import os
34import subprocess
35import sys
36import threading
37
halcanary20fb7c62014-06-25 13:28:29 -070038
39def git_executable():
40 """Find the git executable.
41
42 Returns:
43 A string suitable for passing to subprocess functions, or None.
44 """
45 envgit = os.environ.get('GIT_EXECUTABLE')
Ben Wagner96aa5352018-06-19 10:34:32 -040046 searchlist = ['git']
halcanary20fb7c62014-06-25 13:28:29 -070047 if envgit:
48 searchlist.insert(0, envgit)
49 with open(os.devnull, 'w') as devnull:
50 for git in searchlist:
51 try:
52 subprocess.call([git, '--version'], stdout=devnull)
53 except (OSError,):
54 continue
55 return git
56 return None
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +000057
58
59DEFAULT_DEPS_PATH = os.path.normpath(
60 os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS'))
61
62
63def usage(deps_file_path = None):
64 sys.stderr.write(
65 'Usage: run to grab dependencies, with optional platform support:\n')
Hal Canarya6f6f772017-01-27 11:57:13 -050066 sys.stderr.write(' %s %s' % (sys.executable, __file__))
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +000067 if deps_file_path:
Hal Canarya6f6f772017-01-27 11:57:13 -050068 parsed_deps = parse_file_to_dict(deps_file_path)
69 if 'deps_os' in parsed_deps:
70 for deps_os in parsed_deps['deps_os']:
71 sys.stderr.write(' [%s]' % deps_os)
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +000072 sys.stderr.write('\n\n')
73 sys.stderr.write(__doc__)
74
75
76def git_repository_sync_is_disabled(git, directory):
77 try:
78 disable = subprocess.check_output(
79 [git, 'config', 'sync-deps.disable'], cwd=directory)
80 return disable.lower().strip() in ['true', '1', 'yes', 'on']
81 except subprocess.CalledProcessError:
82 return False
83
84
commit-bot@chromium.orgaae6e6f2014-04-21 19:03:05 +000085def is_git_toplevel(git, directory):
86 """Return true iff the directory is the top level of a Git repository.
87
88 Args:
89 git (string) the git executable
90
91 directory (string) the path into which the repository
92 is expected to be checked out.
93 """
94 try:
95 toplevel = subprocess.check_output(
96 [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip()
Hal Canary70a4fd22020-01-09 12:35:22 -050097 return os.path.realpath(directory) == os.path.realpath(toplevel.decode())
commit-bot@chromium.orgaae6e6f2014-04-21 19:03:05 +000098 except subprocess.CalledProcessError:
99 return False
100
101
Hal Canary5e5f9dc2019-11-01 10:34:32 -0400102def status(directory, commithash, change):
Hal Canarya6f6f772017-01-27 11:57:13 -0500103 def truncate(s, length):
104 return s if len(s) <= length else s[:(length - 3)] + '...'
105 dlen = 36
106 directory = truncate(directory, dlen)
Hal Canary5e5f9dc2019-11-01 10:34:32 -0400107 commithash = truncate(commithash, 40)
108 symbol = '>' if change else '@'
109 sys.stdout.write('%-*s %s %s\n' % (dlen, directory, symbol, commithash))
Hal Canarya6f6f772017-01-27 11:57:13 -0500110
111
Hal Canary5e5f9dc2019-11-01 10:34:32 -0400112def git_checkout_to_directory(git, repo, commithash, directory, verbose):
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000113 """Checkout (and clone if needed) a Git repository.
114
115 Args:
116 git (string) the git executable
117
118 repo (string) the location of the repository, suitable
119 for passing to `git clone`.
120
Hal Canary5e5f9dc2019-11-01 10:34:32 -0400121 commithash (string) a commit, suitable for passing to `git checkout`
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000122
123 directory (string) the path into which the repository
124 should be checked out.
125
126 verbose (boolean)
127
128 Raises an exception if any calls to git fail.
129 """
130 if not os.path.isdir(directory):
131 subprocess.check_call(
Hal Canary5e5f9dc2019-11-01 10:34:32 -0400132 [git, 'clone', '--quiet', '--no-checkout', repo, directory])
133 subprocess.check_call([git, 'checkout', '--quiet', commithash],
134 cwd=directory)
135 if verbose:
136 status(directory, commithash, True)
137 return
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000138
commit-bot@chromium.orgaae6e6f2014-04-21 19:03:05 +0000139 if not is_git_toplevel(git, directory):
140 # if the directory exists, but isn't a git repo, you will modify
141 # the parent repostory, which isn't what you want.
142 sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory)
143 return
144
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000145 # Check to see if this repo is disabled. Quick return.
146 if git_repository_sync_is_disabled(git, directory):
147 sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory)
148 return
149
Hal Canary73eae982017-02-13 13:40:56 -0500150 with open(os.devnull, 'w') as devnull:
151 # If this fails, we will fetch before trying again. Don't spam user
152 # with error infomation.
Hal Canary5e5f9dc2019-11-01 10:34:32 -0400153 if 0 == subprocess.call([git, 'checkout', '--quiet', commithash],
Hal Canary73eae982017-02-13 13:40:56 -0500154 cwd=directory, stderr=devnull):
155 # if this succeeds, skip slow `git fetch`.
156 if verbose:
Hal Canary5e5f9dc2019-11-01 10:34:32 -0400157 status(directory, commithash, False) # Success.
Hal Canary73eae982017-02-13 13:40:56 -0500158 return
halcanary6aff54c2015-11-03 09:50:03 -0800159
Hal Canarya6f6f772017-01-27 11:57:13 -0500160 # If the repo has changed, always force use of the correct repo.
161 # If origin already points to repo, this is a quick no-op.
162 subprocess.check_call(
163 [git, 'remote', 'set-url', 'origin', repo], cwd=directory)
164
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000165 subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory)
166
Hal Canary5e5f9dc2019-11-01 10:34:32 -0400167 subprocess.check_call([git, 'checkout', '--quiet', commithash], cwd=directory)
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000168
169 if verbose:
Hal Canary5e5f9dc2019-11-01 10:34:32 -0400170 status(directory, commithash, True) # Success.
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000171
172
173def parse_file_to_dict(path):
174 dictionary = {}
Hal Canary70a4fd22020-01-09 12:35:22 -0500175 with open(path) as f:
176 exec(f.read(), dictionary)
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000177 return dictionary
178
179
Hal Canary5e5f9dc2019-11-01 10:34:32 -0400180def is_sha1_sum(s):
181 """SHA1 sums are 160 bits, encoded as lowercase hexadecimal."""
182 return len(s) == 40 and all(c in '0123456789abcdef' for c in s)
183
184
halcanaryc6c06242014-08-26 12:06:47 -0700185def git_sync_deps(deps_file_path, command_line_os_requests, verbose):
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000186 """Grab dependencies, with optional platform support.
187
188 Args:
189 deps_file_path (string) Path to the DEPS file.
190
halcanaryc6c06242014-08-26 12:06:47 -0700191 command_line_os_requests (list of strings) Can be empty list.
192 List of strings that should each be a key in the deps_os
193 dictionary in the DEPS file.
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000194
halcanaryc6c06242014-08-26 12:06:47 -0700195 Raises git Exceptions.
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000196 """
halcanary20fb7c62014-06-25 13:28:29 -0700197 git = git_executable()
198 assert git
199
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000200 deps_file_directory = os.path.dirname(deps_file_path)
halcanaryc6c06242014-08-26 12:06:47 -0700201 deps_file = parse_file_to_dict(deps_file_path)
202 dependencies = deps_file['deps'].copy()
Hal Canarya6f6f772017-01-27 11:57:13 -0500203 os_specific_dependencies = deps_file.get('deps_os', dict())
204 if 'all' in command_line_os_requests:
205 for value in os_specific_dependencies.itervalues():
206 dependencies.update(value)
207 else:
208 for os_name in command_line_os_requests:
209 # Add OS-specific dependencies
210 if os_name in os_specific_dependencies:
211 dependencies.update(os_specific_dependencies[os_name])
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000212 for directory in dependencies:
Hal Canarya6f6f772017-01-27 11:57:13 -0500213 for other_dir in dependencies:
214 if directory.startswith(other_dir + '/'):
215 raise Exception('%r is parent of %r' % (other_dir, directory))
216 list_of_arg_lists = []
217 for directory in sorted(dependencies):
Hal Canary70a4fd22020-01-09 12:35:22 -0500218 if not isinstance(dependencies[directory], str):
Hal Canary2c74c322018-06-20 10:28:37 -0400219 if verbose:
Hal Canary70a4fd22020-01-09 12:35:22 -0500220 sys.stdout.write( 'Skipping "%s".\n' % directory)
Hal Canary2c74c322018-06-20 10:28:37 -0400221 continue
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000222 if '@' in dependencies[directory]:
Hal Canary5e5f9dc2019-11-01 10:34:32 -0400223 repo, commithash = dependencies[directory].split('@', 1)
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000224 else:
Hal Canary5e5f9dc2019-11-01 10:34:32 -0400225 raise Exception("please specify commit")
226 if not is_sha1_sum(commithash):
227 raise Exception("poorly formed commit hash: %r" % commithash)
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000228
229 relative_directory = os.path.join(deps_file_directory, directory)
230
231 list_of_arg_lists.append(
Hal Canary5e5f9dc2019-11-01 10:34:32 -0400232 (git, repo, commithash, relative_directory, verbose))
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000233
234 multithread(git_checkout_to_directory, list_of_arg_lists)
235
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000236
237def multithread(function, list_of_arg_lists):
238 # for args in list_of_arg_lists:
239 # function(*args)
240 # return
241 threads = []
242 for args in list_of_arg_lists:
243 thread = threading.Thread(None, function, None, args)
244 thread.start()
245 threads.append(thread)
246 for thread in threads:
247 thread.join()
248
249
250def main(argv):
251 deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH)
252 verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False))
halcanaryc6c06242014-08-26 12:06:47 -0700253
254 if '--help' in argv or '-h' in argv:
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000255 usage(deps_file_path)
256 return 1
257
halcanaryc6c06242014-08-26 12:06:47 -0700258 git_sync_deps(deps_file_path, argv, verbose)
Hal Canarya6f6f772017-01-27 11:57:13 -0500259 subprocess.check_call(
260 [sys.executable,
261 os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')])
halcanaryc6c06242014-08-26 12:06:47 -0700262 return 0
263
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000264
265if __name__ == '__main__':
266 exit(main(sys.argv[1:]))