blob: 1b716351891b33bbd0b2c6e6f8b39d1da3791ae1 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
David Pursehouse86d973d2012-08-24 10:21:02 +090016import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070017from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070018import os
19import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070020import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070021import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070022import subprocess
23import sys
Shawn O. Pearcef6906872009-04-18 10:49:00 -070024import time
David Pursehouse86d973d2012-08-24 10:21:02 +090025import urlparse
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070026import xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070027
Roy Lee18afd7f2010-05-09 04:32:08 +080028try:
29 import threading as _threading
30except ImportError:
31 import dummy_threading as _threading
32
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070033try:
34 import resource
35 def _rlimit_nofile():
36 return resource.getrlimit(resource.RLIMIT_NOFILE)
37except ImportError:
38 def _rlimit_nofile():
39 return (256, 256)
40
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070041from git_command import GIT
David Pursehoused94aaef2012-09-07 09:52:04 +090042from git_refs import R_HEADS, HEAD
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070043from project import Project
44from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080045from command import Command, MirrorSafeCommand
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070046from error import RepoChangedException, GitError
Shawn O. Pearce350cde42009-04-16 11:21:18 -070047from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070048from progress import Progress
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070049
Doug Andersonfc06ced2011-03-16 15:49:18 -070050class _FetchError(Exception):
51 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
52 pass
53
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080054class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080055 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070056 common = True
57 helpSummary = "Update working tree to the latest revision"
58 helpUsage = """
59%prog [<project>...]
60"""
61 helpDescription = """
62The '%prog' command synchronizes local project directories
63with the remote repositories specified in the manifest. If a local
64project does not yet exist, it will clone a new local directory from
65the remote repository and set up tracking branches as specified in
66the manifest. If the local project already exists, '%prog'
67will update the remote branches and rebase any new local changes
68on top of the new remote changes.
69
70'%prog' will synchronize all projects listed at the command
71line. Projects can be specified either by name, or by a relative
72or absolute path to the project's local directory. If no projects
73are specified, '%prog' will synchronize all projects listed in
74the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070075
76The -d/--detach option can be used to switch specified projects
77back to the manifest revision. This option is especially helpful
78if the project is currently on a topic branch, but the manifest
79revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -070080
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070081The -s/--smart-sync option can be used to sync to a known good
82build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +020083manifest. The -t/--smart-tag option is similar and allows you to
84specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070085
David Pursehousecf76b1b2012-09-14 10:31:42 +090086The -u/--manifest-server-username and -p/--manifest-server-password
87options can be used to specify a username and password to authenticate
88with the manifest server when using the -s or -t option.
89
90If -u and -p are not specified when using the -s or -t option, '%prog'
91will attempt to read authentication credentials for the manifest server
92from the user's .netrc file.
93
94'%prog' will not use authentication credentials from -u/-p or .netrc
95if the manifest server specified in the manifest file already includes
96credentials.
97
Andrei Warkentin5df6de02010-07-02 17:58:31 -050098The -f/--force-broken option can be used to proceed with syncing
99other projects if a project sync fails.
100
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700101The --no-clone-bundle option disables any attempt to use
102$URL/clone.bundle to bootstrap a new Git repository from a
103resumeable bundle file on a content delivery network. This
104may be necessary if there are problems with the local Python
105HTTP client or proxy configuration, but the Git binary works.
106
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700107SSH Connections
108---------------
109
110If at least one project remote URL uses an SSH connection (ssh://,
111git+ssh://, or user@host:path syntax) repo will automatically
112enable the SSH ControlMaster option when connecting to that host.
113This feature permits other projects in the same '%prog' session to
114reuse the same SSH tunnel, saving connection setup overheads.
115
116To disable this behavior on UNIX platforms, set the GIT_SSH
117environment variable to 'ssh'. For example:
118
119 export GIT_SSH=ssh
120 %prog
121
122Compatibility
123~~~~~~~~~~~~~
124
125This feature is automatically disabled on Windows, due to the lack
126of UNIX domain socket support.
127
128This feature is not compatible with url.insteadof rewrites in the
129user's ~/.gitconfig. '%prog' is currently not able to perform the
130rewrite early enough to establish the ControlMaster tunnel.
131
132If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
133later is required to fix a server side protocol bug.
134
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700135"""
136
Nico Sallembien6623b212010-05-11 12:57:01 -0700137 def _Options(self, p, show_smart=True):
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700138 self.jobs = self.manifest.default.sync_j
139
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500140 p.add_option('-f', '--force-broken',
141 dest='force_broken', action='store_true',
142 help="continue sync even if a project fails to sync")
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700143 p.add_option('-l','--local-only',
144 dest='local_only', action='store_true',
145 help="only update working tree, don't fetch")
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700146 p.add_option('-n','--network-only',
147 dest='network_only', action='store_true',
148 help="fetch only, don't update working tree")
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700149 p.add_option('-d','--detach',
150 dest='detach_head', action='store_true',
151 help='detach projects back to manifest revision')
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700152 p.add_option('-c','--current-branch',
153 dest='current_branch_only', action='store_true',
154 help='fetch only current branch from server')
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700155 p.add_option('-q','--quiet',
156 dest='quiet', action='store_true',
157 help='be more quiet')
Roy Lee18afd7f2010-05-09 04:32:08 +0800158 p.add_option('-j','--jobs',
159 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700160 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500161 p.add_option('-m', '--manifest-name',
162 dest='manifest_name',
163 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700164 p.add_option('--no-clone-bundle',
165 dest='no_clone_bundle', action='store_true',
166 help='disable use of /clone.bundle on HTTP/HTTPS')
Nico Sallembien6623b212010-05-11 12:57:01 -0700167 if show_smart:
168 p.add_option('-s', '--smart-sync',
169 dest='smart_sync', action='store_true',
170 help='smart sync using manifest from a known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200171 p.add_option('-t', '--smart-tag',
172 dest='smart_tag', action='store',
173 help='smart sync using manifest from a known tag')
David Pursehousecf76b1b2012-09-14 10:31:42 +0900174 p.add_option('-u', '--manifest-server-username', action='store',
175 dest='manifest_server_username',
176 help='username to authenticate with the manifest server')
177 p.add_option('-p', '--manifest-server-password', action='store',
178 dest='manifest_server_password',
179 help='password to authenticate with the manifest server')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700180
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700181 g = p.add_option_group('repo Version options')
182 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700183 dest='no_repo_verify', action='store_true',
184 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700185 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800186 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700187 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700188
Doug Andersonfc06ced2011-03-16 15:49:18 -0700189 def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
190 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800191
Doug Andersonfc06ced2011-03-16 15:49:18 -0700192 Args:
193 opt: Program options returned from optparse. See _Options().
194 project: Project object for the project to fetch.
195 lock: Lock for accessing objects that are shared amongst multiple
196 _FetchHelper() threads.
197 fetched: set object that we will add project.gitdir to when we're done
198 (with our lock held).
199 pm: Instance of a Project object. We will call pm.update() (with our
200 lock held).
201 sem: We'll release() this semaphore when we exit so that another thread
202 can be started up.
203 err_event: We'll set this event in the case of an error (after printing
204 out info about the error).
205 """
206 # We'll set to true once we've locked the lock.
207 did_lock = False
208
209 # Encapsulate everything in a try/except/finally so that:
210 # - We always set err_event in the case of an exception.
211 # - We always make sure we call sem.release().
212 # - We always make sure we unlock the lock if we locked it.
213 try:
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700214 try:
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700215 success = project.Sync_NetworkHalf(
216 quiet=opt.quiet,
217 current_branch_only=opt.current_branch_only,
218 clone_bundle=not opt.no_clone_bundle)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700219
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700220 # Lock around all the rest of the code, since printing, updating a set
221 # and Progress.update() are not thread safe.
222 lock.acquire()
223 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700224
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700225 if not success:
226 print >>sys.stderr, 'error: Cannot fetch %s' % project.name
227 if opt.force_broken:
228 print >>sys.stderr, 'warn: --force-broken, continuing to sync'
229 else:
230 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700231
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700232 fetched.add(project.gitdir)
233 pm.update()
Shawn O. Pearcedf5ee522011-10-11 14:05:21 -0700234 except _FetchError:
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700235 err_event.set()
Shawn O. Pearcedf5ee522011-10-11 14:05:21 -0700236 except:
237 err_event.set()
238 raise
Doug Andersonfc06ced2011-03-16 15:49:18 -0700239 finally:
240 if did_lock:
241 lock.release()
242 sem.release()
Roy Lee18afd7f2010-05-09 04:32:08 +0800243
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700244 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700245 fetched = set()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700246 pm = Progress('Fetching projects', len(projects))
Roy Lee18afd7f2010-05-09 04:32:08 +0800247
248 if self.jobs == 1:
249 for project in projects:
250 pm.update()
Shawn O. Pearce5d0efdb2012-08-02 12:13:01 -0700251 if project.Sync_NetworkHalf(
252 quiet=opt.quiet,
253 current_branch_only=opt.current_branch_only,
254 clone_bundle=not opt.no_clone_bundle):
Roy Lee18afd7f2010-05-09 04:32:08 +0800255 fetched.add(project.gitdir)
256 else:
257 print >>sys.stderr, 'error: Cannot fetch %s' % project.name
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500258 if opt.force_broken:
259 print >>sys.stderr, 'warn: --force-broken, continuing to sync'
260 else:
261 sys.exit(1)
Roy Lee18afd7f2010-05-09 04:32:08 +0800262 else:
263 threads = set()
264 lock = _threading.Lock()
265 sem = _threading.Semaphore(self.jobs)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700266 err_event = _threading.Event()
Roy Lee18afd7f2010-05-09 04:32:08 +0800267 for project in projects:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700268 # Check for any errors before starting any new threads.
269 # ...we'll let existing threads finish, though.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400270 if err_event.isSet():
Doug Andersonfc06ced2011-03-16 15:49:18 -0700271 break
272
Roy Lee18afd7f2010-05-09 04:32:08 +0800273 sem.acquire()
274 t = _threading.Thread(target = self._FetchHelper,
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700275 args = (opt,
276 project,
277 lock,
278 fetched,
279 pm,
Doug Andersonfc06ced2011-03-16 15:49:18 -0700280 sem,
281 err_event))
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200282 # Ensure that Ctrl-C will not freeze the repo process.
283 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800284 threads.add(t)
285 t.start()
286
287 for t in threads:
288 t.join()
289
Doug Andersonfc06ced2011-03-16 15:49:18 -0700290 # If we saw an error, exit with code 1 so that other scripts can check.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400291 if err_event.isSet():
Doug Andersonfc06ced2011-03-16 15:49:18 -0700292 print >>sys.stderr, '\nerror: Exited sync due to fetch errors'
293 sys.exit(1)
294
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700295 pm.end()
Shawn O. Pearce0d2b61f2009-07-03 15:22:49 -0700296 for project in projects:
297 project.bare_git.gc('--auto')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700298 return fetched
299
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700300 def UpdateProjectList(self):
301 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700302 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700303 if project.relpath:
304 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700305 file_name = 'project.list'
306 file_path = os.path.join(self.manifest.repodir, file_name)
307 old_project_paths = []
308
309 if os.path.exists(file_path):
310 fd = open(file_path, 'r')
311 try:
312 old_project_paths = fd.read().split('\n')
313 finally:
314 fd.close()
315 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700316 if not path:
317 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700318 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900319 # If the path has already been deleted, we don't need to do it
Anthonyf3fdf822009-09-26 13:38:52 -0400320 if os.path.exists(self.manifest.topdir + '/' + path):
321 project = Project(
322 manifest = self.manifest,
323 name = path,
324 remote = RemoteSpec('origin'),
325 gitdir = os.path.join(self.manifest.topdir,
326 path, '.git'),
327 worktree = os.path.join(self.manifest.topdir, path),
328 relpath = path,
329 revisionExpr = 'HEAD',
Colin Cross5acde752012-03-28 20:15:45 -0700330 revisionId = None,
331 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400332
333 if project.IsDirty():
334 print >>sys.stderr, 'error: Cannot remove project "%s": \
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700335uncommitted changes are present' % project.relpath
Anthonyf3fdf822009-09-26 13:38:52 -0400336 print >>sys.stderr, ' commit changes, then run sync again'
337 return -1
338 else:
339 print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
340 shutil.rmtree(project.worktree)
341 # Try deleting parent subdirs if they are empty
342 dir = os.path.dirname(project.worktree)
343 while dir != self.manifest.topdir:
344 try:
345 os.rmdir(dir)
346 except OSError:
347 break
348 dir = os.path.dirname(dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700349
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700350 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700351 fd = open(file_path, 'w')
352 try:
353 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700354 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700355 finally:
356 fd.close()
357 return 0
358
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700359 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800360 if opt.jobs:
361 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700362 if self.jobs > 1:
363 soft_limit, _ = _rlimit_nofile()
364 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
365
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700366 if opt.network_only and opt.detach_head:
367 print >>sys.stderr, 'error: cannot combine -n and -d'
368 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700369 if opt.network_only and opt.local_only:
370 print >>sys.stderr, 'error: cannot combine -n and -l'
371 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500372 if opt.manifest_name and opt.smart_sync:
373 print >>sys.stderr, 'error: cannot combine -m and -s'
374 sys.exit(1)
375 if opt.manifest_name and opt.smart_tag:
376 print >>sys.stderr, 'error: cannot combine -m and -t'
377 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900378 if opt.manifest_server_username or opt.manifest_server_password:
379 if not (opt.smart_sync or opt.smart_tag):
380 print >>sys.stderr, 'error: -u and -p may only be combined with ' \
381 '-s or -t'
382 sys.exit(1)
383 if None in [opt.manifest_server_username, opt.manifest_server_password]:
384 print >>sys.stderr, 'error: both -u and -p must be given'
385 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500386
387 if opt.manifest_name:
388 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700389
Victor Boivie08c880d2011-04-19 10:32:52 +0200390 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700391 if not self.manifest.manifest_server:
392 print >>sys.stderr, \
393 'error: cannot smart sync: no manifest server defined in manifest'
394 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900395
396 manifest_server = self.manifest.manifest_server
David Pursehousecf76b1b2012-09-14 10:31:42 +0900397
David Pursehouse86d973d2012-08-24 10:21:02 +0900398 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900399 username = None
400 password = None
401 if opt.manifest_server_username and opt.manifest_server_password:
402 username = opt.manifest_server_username
403 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900404 else:
405 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900406 info = netrc.netrc()
407 except IOError:
408 print >>sys.stderr, '.netrc file does not exist or could not be opened'
David Pursehouse86d973d2012-08-24 10:21:02 +0900409 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900410 try:
411 parse_result = urlparse.urlparse(manifest_server)
412 if parse_result.hostname:
413 username, _account, password = \
414 info.authenticators(parse_result.hostname)
415 except TypeError:
416 # TypeError is raised when the given hostname is not present
417 # in the .netrc file.
418 print >>sys.stderr, 'No credentials found for %s in .netrc' % \
419 parse_result.hostname
420 except netrc.NetrcParseError, e:
421 print >>sys.stderr, 'Error parsing .netrc file: %s' % e
422
423 if (username and password):
424 manifest_server = manifest_server.replace('://', '://%s:%s@' %
425 (username, password),
426 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900427
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700428 try:
David Pursehouse86d973d2012-08-24 10:21:02 +0900429 server = xmlrpclib.Server(manifest_server)
Victor Boivie08c880d2011-04-19 10:32:52 +0200430 if opt.smart_sync:
431 p = self.manifest.manifestProject
432 b = p.GetBranch(p.CurrentBranch)
433 branch = b.merge
434 if branch.startswith(R_HEADS):
435 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700436
Victor Boivie08c880d2011-04-19 10:32:52 +0200437 env = os.environ.copy()
438 if (env.has_key('TARGET_PRODUCT') and
439 env.has_key('TARGET_BUILD_VARIANT')):
440 target = '%s-%s' % (env['TARGET_PRODUCT'],
441 env['TARGET_BUILD_VARIANT'])
442 [success, manifest_str] = server.GetApprovedManifest(branch, target)
443 else:
444 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700445 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200446 assert(opt.smart_tag)
447 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700448
449 if success:
450 manifest_name = "smart_sync_override.xml"
451 manifest_path = os.path.join(self.manifest.manifestProject.worktree,
452 manifest_name)
453 try:
454 f = open(manifest_path, 'w')
455 try:
456 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700457 finally:
458 f.close()
459 except IOError:
460 print >>sys.stderr, 'error: cannot write manifest to %s' % \
461 manifest_path
462 sys.exit(1)
Nico Sallembien719965a2010-04-20 15:28:19 -0700463 self.manifest.Override(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700464 else:
465 print >>sys.stderr, 'error: %s' % manifest_str
466 sys.exit(1)
David Pursehousebd489c42012-08-23 10:21:26 +0900467 except (socket.error, IOError, xmlrpclib.Fault), e:
468 print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%s' % (
469 self.manifest.manifest_server, e)
470 sys.exit(1)
471 except xmlrpclib.ProtocolError, e:
472 print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%d %s' % (
473 self.manifest.manifest_server, e.errcode, e.errmsg)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700474 sys.exit(1)
475
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700476 rp = self.manifest.repoProject
477 rp.PreSync()
478
479 mp = self.manifest.manifestProject
480 mp.PreSync()
481
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800482 if opt.repo_upgraded:
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700483 _PostRepoUpgrade(self.manifest)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800484
Nico Sallembien9bb18162009-12-07 15:38:01 -0800485 if not opt.local_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700486 mp.Sync_NetworkHalf(quiet=opt.quiet,
487 current_branch_only=opt.current_branch_only)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800488
489 if mp.HasChanges:
490 syncbuf = SyncBuffer(mp.config)
491 mp.Sync_LocalHalf(syncbuf)
492 if not syncbuf.Finish():
493 sys.exit(1)
494 self.manifest._Unload()
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700495 if opt.jobs is None:
496 self.jobs = self.manifest.default.sync_j
David Pursehouse8a68ff92012-09-24 12:15:13 +0900497 all_projects = self.GetProjects(args, missing_ok=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700498
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700499 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700500 to_fetch = []
501 now = time.time()
502 if (24 * 60 * 60) <= (now - rp.LastFetch):
503 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900504 to_fetch.extend(all_projects)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700505
David Pursehouse8a68ff92012-09-24 12:15:13 +0900506 self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700507 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700508 if opt.network_only:
509 # bail out now; the rest touches the working tree
510 return
511
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700512 if self.manifest.IsMirror:
513 # bail out now, we have no working tree
514 return
515
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700516 if self.UpdateProjectList():
517 sys.exit(1)
518
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700519 syncbuf = SyncBuffer(mp.config,
520 detach_head = opt.detach_head)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900521 pm = Progress('Syncing work tree', len(all_projects))
522 for project in all_projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700523 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800524 if project.worktree:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700525 project.Sync_LocalHalf(syncbuf)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700526 pm.end()
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700527 print >>sys.stderr
528 if not syncbuf.Finish():
529 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700530
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700531 # If there's a notice that's supposed to print at the end of the sync, print
532 # it now...
533 if self.manifest.notice:
534 print self.manifest.notice
535
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700536def _PostRepoUpgrade(manifest):
537 for project in manifest.projects.values():
538 if project.Exists:
539 project.PostRepoUpgrade()
540
541def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
542 if rp.HasChanges:
543 print >>sys.stderr, 'info: A new version of repo is available'
544 print >>sys.stderr, ''
545 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700546 syncbuf = SyncBuffer(rp.config)
547 rp.Sync_LocalHalf(syncbuf)
548 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700549 sys.exit(1)
550 print >>sys.stderr, 'info: Restarting repo with latest version'
551 raise RepoChangedException(['--repo-upgraded'])
552 else:
553 print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
554 else:
555 if verbose:
556 print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
557
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700558def _VerifyTag(project):
559 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
560 if not os.path.exists(gpg_dir):
561 print >>sys.stderr,\
562"""warning: GnuPG was not available during last "repo init"
563warning: Cannot automatically authenticate repo."""
564 return True
565
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700566 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700567 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700568 except GitError:
569 cur = None
570
571 if not cur \
572 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700573 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700574 if rev.startswith(R_HEADS):
575 rev = rev[len(R_HEADS):]
576
577 print >>sys.stderr
578 print >>sys.stderr,\
579 "warning: project '%s' branch '%s' is not signed" \
580 % (project.name, rev)
581 return False
582
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800583 env = os.environ.copy()
584 env['GIT_DIR'] = project.gitdir.encode()
585 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700586
587 cmd = [GIT, 'tag', '-v', cur]
588 proc = subprocess.Popen(cmd,
589 stdout = subprocess.PIPE,
590 stderr = subprocess.PIPE,
591 env = env)
592 out = proc.stdout.read()
593 proc.stdout.close()
594
595 err = proc.stderr.read()
596 proc.stderr.close()
597
598 if proc.wait() != 0:
599 print >>sys.stderr
600 print >>sys.stderr, out
601 print >>sys.stderr, err
602 print >>sys.stderr
603 return False
604 return True