blob: 78449d9682e62e86098bd490e3e69985661b8c44 [file] [log] [blame]
Hal Canarya6f6f772017-01-27 11:57:13 -05001#!/usr/bin/env python
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +00002# Copyright 2014 Google Inc.
3#
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7
8"""Parse a DEPS file and git checkout all of the dependencies.
9
10Args:
11 An optional list of deps_os values.
12
13Environment Variables:
halcanary20fb7c62014-06-25 13:28:29 -070014 GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of
15 ['git', 'git.exe', 'git.bat'] in your default path.
16
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +000017 GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset,
18 will use the file ../DEPS relative to this script's directory.
19
20 GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages.
21
22Git Config:
23 To disable syncing of a single repository:
24 cd path/to/repository
25 git config sync-deps.disable true
26
27 To re-enable sync:
28 cd path/to/repository
29 git config --unset sync-deps.disable
30"""
31
32
33import os
34import subprocess
35import sys
36import threading
37
halcanary20fb7c62014-06-25 13:28:29 -070038
39def git_executable():
40 """Find the git executable.
41
42 Returns:
43 A string suitable for passing to subprocess functions, or None.
44 """
45 envgit = os.environ.get('GIT_EXECUTABLE')
46 searchlist = ['git', 'git.exe', 'git.bat']
47 if envgit:
48 searchlist.insert(0, envgit)
49 with open(os.devnull, 'w') as devnull:
50 for git in searchlist:
51 try:
52 subprocess.call([git, '--version'], stdout=devnull)
53 except (OSError,):
54 continue
55 return git
56 return None
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +000057
58
59DEFAULT_DEPS_PATH = os.path.normpath(
60 os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS'))
61
62
63def usage(deps_file_path = None):
64 sys.stderr.write(
65 'Usage: run to grab dependencies, with optional platform support:\n')
Hal Canarya6f6f772017-01-27 11:57:13 -050066 sys.stderr.write(' %s %s' % (sys.executable, __file__))
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +000067 if deps_file_path:
Hal Canarya6f6f772017-01-27 11:57:13 -050068 parsed_deps = parse_file_to_dict(deps_file_path)
69 if 'deps_os' in parsed_deps:
70 for deps_os in parsed_deps['deps_os']:
71 sys.stderr.write(' [%s]' % deps_os)
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +000072 sys.stderr.write('\n\n')
73 sys.stderr.write(__doc__)
74
75
76def git_repository_sync_is_disabled(git, directory):
77 try:
78 disable = subprocess.check_output(
79 [git, 'config', 'sync-deps.disable'], cwd=directory)
80 return disable.lower().strip() in ['true', '1', 'yes', 'on']
81 except subprocess.CalledProcessError:
82 return False
83
84
commit-bot@chromium.orgaae6e6f2014-04-21 19:03:05 +000085def is_git_toplevel(git, directory):
86 """Return true iff the directory is the top level of a Git repository.
87
88 Args:
89 git (string) the git executable
90
91 directory (string) the path into which the repository
92 is expected to be checked out.
93 """
94 try:
95 toplevel = subprocess.check_output(
96 [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip()
commit-bot@chromium.org2349a1e2014-04-28 13:39:47 +000097 return os.path.realpath(directory) == os.path.realpath(toplevel)
commit-bot@chromium.orgaae6e6f2014-04-21 19:03:05 +000098 except subprocess.CalledProcessError:
99 return False
100
101
Hal Canarya6f6f772017-01-27 11:57:13 -0500102def status(directory, checkoutable):
103 def truncate(s, length):
104 return s if len(s) <= length else s[:(length - 3)] + '...'
105 dlen = 36
106 directory = truncate(directory, dlen)
107 checkoutable = truncate(checkoutable, 40)
108 sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable))
109
110
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000111def git_checkout_to_directory(git, repo, checkoutable, directory, verbose):
112 """Checkout (and clone if needed) a Git repository.
113
114 Args:
115 git (string) the git executable
116
117 repo (string) the location of the repository, suitable
118 for passing to `git clone`.
119
120 checkoutable (string) a tag, branch, or commit, suitable for
121 passing to `git checkout`
122
123 directory (string) the path into which the repository
124 should be checked out.
125
126 verbose (boolean)
127
128 Raises an exception if any calls to git fail.
129 """
130 if not os.path.isdir(directory):
131 subprocess.check_call(
132 [git, 'clone', '--quiet', repo, directory])
133
commit-bot@chromium.orgaae6e6f2014-04-21 19:03:05 +0000134 if not is_git_toplevel(git, directory):
135 # if the directory exists, but isn't a git repo, you will modify
136 # the parent repostory, which isn't what you want.
137 sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory)
138 return
139
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000140 # Check to see if this repo is disabled. Quick return.
141 if git_repository_sync_is_disabled(git, directory):
142 sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory)
143 return
144
Hal Canary73eae982017-02-13 13:40:56 -0500145 with open(os.devnull, 'w') as devnull:
146 # If this fails, we will fetch before trying again. Don't spam user
147 # with error infomation.
148 if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable],
149 cwd=directory, stderr=devnull):
150 # if this succeeds, skip slow `git fetch`.
151 if verbose:
152 status(directory, checkoutable) # Success.
153 return
halcanary6aff54c2015-11-03 09:50:03 -0800154
Hal Canarya6f6f772017-01-27 11:57:13 -0500155 # If the repo has changed, always force use of the correct repo.
156 # If origin already points to repo, this is a quick no-op.
157 subprocess.check_call(
158 [git, 'remote', 'set-url', 'origin', repo], cwd=directory)
159
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000160 subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory)
161
Hal Canary49deb7e2017-02-07 12:05:47 -0500162 subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory)
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000163
164 if verbose:
Hal Canarya6f6f772017-01-27 11:57:13 -0500165 status(directory, checkoutable) # Success.
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000166
167
168def parse_file_to_dict(path):
169 dictionary = {}
170 execfile(path, dictionary)
171 return dictionary
172
173
halcanaryc6c06242014-08-26 12:06:47 -0700174def git_sync_deps(deps_file_path, command_line_os_requests, verbose):
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000175 """Grab dependencies, with optional platform support.
176
177 Args:
178 deps_file_path (string) Path to the DEPS file.
179
halcanaryc6c06242014-08-26 12:06:47 -0700180 command_line_os_requests (list of strings) Can be empty list.
181 List of strings that should each be a key in the deps_os
182 dictionary in the DEPS file.
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000183
halcanaryc6c06242014-08-26 12:06:47 -0700184 Raises git Exceptions.
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000185 """
halcanary20fb7c62014-06-25 13:28:29 -0700186 git = git_executable()
187 assert git
188
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000189 deps_file_directory = os.path.dirname(deps_file_path)
halcanaryc6c06242014-08-26 12:06:47 -0700190 deps_file = parse_file_to_dict(deps_file_path)
191 dependencies = deps_file['deps'].copy()
Hal Canarya6f6f772017-01-27 11:57:13 -0500192 os_specific_dependencies = deps_file.get('deps_os', dict())
193 if 'all' in command_line_os_requests:
194 for value in os_specific_dependencies.itervalues():
195 dependencies.update(value)
196 else:
197 for os_name in command_line_os_requests:
198 # Add OS-specific dependencies
199 if os_name in os_specific_dependencies:
200 dependencies.update(os_specific_dependencies[os_name])
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000201 for directory in dependencies:
Hal Canarya6f6f772017-01-27 11:57:13 -0500202 for other_dir in dependencies:
203 if directory.startswith(other_dir + '/'):
204 raise Exception('%r is parent of %r' % (other_dir, directory))
205 list_of_arg_lists = []
206 for directory in sorted(dependencies):
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000207 if '@' in dependencies[directory]:
208 repo, checkoutable = dependencies[directory].split('@', 1)
209 else:
halcanary6aff54c2015-11-03 09:50:03 -0800210 raise Exception("please specify commit or tag")
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000211
212 relative_directory = os.path.join(deps_file_directory, directory)
213
214 list_of_arg_lists.append(
halcanary20fb7c62014-06-25 13:28:29 -0700215 (git, repo, checkoutable, relative_directory, verbose))
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000216
217 multithread(git_checkout_to_directory, list_of_arg_lists)
218
halcanaryc6c06242014-08-26 12:06:47 -0700219 for directory in deps_file.get('recursedeps', []):
halcanary5164a972014-08-21 13:17:43 -0700220 recursive_path = os.path.join(deps_file_directory, directory, 'DEPS')
halcanaryc6c06242014-08-26 12:06:47 -0700221 git_sync_deps(recursive_path, command_line_os_requests, verbose)
halcanary5164a972014-08-21 13:17:43 -0700222
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000223
224def multithread(function, list_of_arg_lists):
225 # for args in list_of_arg_lists:
226 # function(*args)
227 # return
228 threads = []
229 for args in list_of_arg_lists:
230 thread = threading.Thread(None, function, None, args)
231 thread.start()
232 threads.append(thread)
233 for thread in threads:
234 thread.join()
235
236
237def main(argv):
238 deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH)
239 verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False))
halcanaryc6c06242014-08-26 12:06:47 -0700240
241 if '--help' in argv or '-h' in argv:
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000242 usage(deps_file_path)
243 return 1
244
halcanaryc6c06242014-08-26 12:06:47 -0700245 git_sync_deps(deps_file_path, argv, verbose)
Hal Canarya6f6f772017-01-27 11:57:13 -0500246 subprocess.check_call(
247 [sys.executable,
248 os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')])
halcanaryc6c06242014-08-26 12:06:47 -0700249 return 0
250
commit-bot@chromium.org71e800a2014-04-16 19:21:00 +0000251
252if __name__ == '__main__':
253 exit(main(sys.argv[1:]))