blob: 9d5874a286a6a0f8b443452e4f13b9815ff1f5ad [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
Chirayu Desai217ea7d2013-03-01 19:14:38 +053017
Dan Willemsen0745bb22015-08-17 13:41:45 -070018import contextlib
19import errno
Anthony King85b24ac2014-05-06 15:57:48 +010020import json
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import os
22import re
Łukasz Gardońbed59ce2017-08-08 10:18:11 +020023import ssl
Shawn O. Pearcefb231612009-04-10 18:53:46 -070024import subprocess
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070025import sys
Doug Anderson0048b692010-12-21 13:39:23 -080026try:
27 import threading as _threading
28except ImportError:
29 import dummy_threading as _threading
Shawn O. Pearcefb231612009-04-10 18:53:46 -070030import time
David Pursehouse59bbb582013-05-17 10:49:33 +090031
32from pyversion import is_python3
33if is_python3():
Sarah Owens1f7627f2012-10-31 09:21:55 -070034 import urllib.request
35 import urllib.error
36else:
David Pursehouse59bbb582013-05-17 10:49:33 +090037 import urllib2
Sarah Owens1f7627f2012-10-31 09:21:55 -070038 import imp
39 urllib = imp.new_module('urllib')
40 urllib.request = urllib2
41 urllib.error = urllib2
Shawn O. Pearcef00e0ce2009-08-22 18:39:49 -070042
Shawn O. Pearcefb231612009-04-10 18:53:46 -070043from signal import SIGTERM
Shawn O. Pearceb54a3922009-01-05 16:18:58 -080044from error import GitError, UploadError
Shawn O. Pearcead3193a2009-04-18 09:54:51 -070045from trace import Trace
David Pursehouseecf8f2b2013-05-24 12:12:23 +090046if is_python3():
47 from http.client import HTTPException
48else:
49 from httplib import HTTPException
Shawn O. Pearceca8c32c2010-05-11 18:21:33 -070050
51from git_command import GitCommand
52from git_command import ssh_sock
53from git_command import terminate_ssh_clients
Zac Livingston9ead97b2017-06-13 08:29:04 -060054from git_refs import R_CHANGES, R_HEADS, R_TAGS
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070055
David Pursehouse1d947b32012-10-25 12:23:11 +090056ID_RE = re.compile(r'^[0-9a-f]{40}$')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070057
Shawn O. Pearce146fe902009-03-25 14:06:43 -070058REVIEW_CACHE = dict()
59
Zac Livingston9ead97b2017-06-13 08:29:04 -060060def IsChange(rev):
61 return rev.startswith(R_CHANGES)
62
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070063def IsId(rev):
64 return ID_RE.match(rev)
65
Zac Livingston9ead97b2017-06-13 08:29:04 -060066def IsTag(rev):
67 return rev.startswith(R_TAGS)
68
69def IsImmutable(rev):
70 return IsChange(rev) or IsId(rev) or IsTag(rev)
71
Shawn O. Pearcef8e32732009-04-17 11:00:31 -070072def _key(name):
73 parts = name.split('.')
74 if len(parts) < 2:
75 return name.lower()
76 parts[ 0] = parts[ 0].lower()
77 parts[-1] = parts[-1].lower()
78 return '.'.join(parts)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070079
80class GitConfig(object):
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070081 _ForUser = None
82
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070083 @classmethod
84 def ForUser(cls):
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070085 if cls._ForUser is None:
David Pursehouse8a68ff92012-09-24 12:15:13 +090086 cls._ForUser = cls(configfile = os.path.expanduser('~/.gitconfig'))
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070087 return cls._ForUser
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070088
89 @classmethod
90 def ForRepository(cls, gitdir, defaults=None):
David Pursehouse8a68ff92012-09-24 12:15:13 +090091 return cls(configfile = os.path.join(gitdir, 'config'),
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070092 defaults = defaults)
93
Anthony King85b24ac2014-05-06 15:57:48 +010094 def __init__(self, configfile, defaults=None, jsonFile=None):
David Pursehouse8a68ff92012-09-24 12:15:13 +090095 self.file = configfile
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070096 self.defaults = defaults
97 self._cache_dict = None
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070098 self._section_dict = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070099 self._remotes = {}
100 self._branches = {}
Shawn O. Pearce1b34c912009-05-21 18:52:49 -0700101
Anthony King85b24ac2014-05-06 15:57:48 +0100102 self._json = jsonFile
103 if self._json is None:
104 self._json = os.path.join(
Shawn O. Pearce1b34c912009-05-21 18:52:49 -0700105 os.path.dirname(self.file),
Anthony King85b24ac2014-05-06 15:57:48 +0100106 '.repo_' + os.path.basename(self.file) + '.json')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700107
108 def Has(self, name, include_defaults = True):
109 """Return true if this configuration file has the key.
110 """
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700111 if _key(name) in self._cache:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700112 return True
113 if include_defaults and self.defaults:
114 return self.defaults.Has(name, include_defaults = True)
115 return False
116
117 def GetBoolean(self, name):
118 """Returns a boolean from the configuration file.
119 None : The value was not defined, or is not a boolean.
120 True : The value was set to true or yes.
121 False: The value was set to false or no.
122 """
123 v = self.GetString(name)
124 if v is None:
125 return None
126 v = v.lower()
127 if v in ('true', 'yes'):
128 return True
129 if v in ('false', 'no'):
130 return False
131 return None
132
David Pursehouse8a68ff92012-09-24 12:15:13 +0900133 def GetString(self, name, all_keys=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700134 """Get the first value for a key, or None if it is not defined.
135
136 This configuration file is used first, if the key is not
David Pursehouse8a68ff92012-09-24 12:15:13 +0900137 defined or all_keys = True then the defaults are also searched.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700138 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700139 try:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700140 v = self._cache[_key(name)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700141 except KeyError:
142 if self.defaults:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900143 return self.defaults.GetString(name, all_keys = all_keys)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700144 v = []
145
David Pursehouse8a68ff92012-09-24 12:15:13 +0900146 if not all_keys:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700147 if v:
148 return v[0]
149 return None
150
151 r = []
152 r.extend(v)
153 if self.defaults:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900154 r.extend(self.defaults.GetString(name, all_keys = True))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700155 return r
156
157 def SetString(self, name, value):
158 """Set the value(s) for a key.
159 Only this configuration file is modified.
160
161 The supplied value should be either a string,
162 or a list of strings (to store multiple values).
163 """
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700164 key = _key(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700165
166 try:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700167 old = self._cache[key]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700168 except KeyError:
169 old = []
170
171 if value is None:
172 if old:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700173 del self._cache[key]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700174 self._do('--unset-all', name)
175
176 elif isinstance(value, list):
177 if len(value) == 0:
178 self.SetString(name, None)
179
180 elif len(value) == 1:
181 self.SetString(name, value[0])
182
183 elif old != value:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700184 self._cache[key] = list(value)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700185 self._do('--replace-all', name, value[0])
Sarah Owensa6053d52012-11-01 13:36:50 -0700186 for i in range(1, len(value)):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700187 self._do('--add', name, value[i])
188
189 elif len(old) != 1 or old[0] != value:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700190 self._cache[key] = [value]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700191 self._do('--replace-all', name, value)
192
193 def GetRemote(self, name):
194 """Get the remote.$name.* configuration values as an object.
195 """
196 try:
197 r = self._remotes[name]
198 except KeyError:
199 r = Remote(self, name)
200 self._remotes[r.name] = r
201 return r
202
203 def GetBranch(self, name):
204 """Get the branch.$name.* configuration values as an object.
205 """
206 try:
207 b = self._branches[name]
208 except KeyError:
209 b = Branch(self, name)
210 self._branches[b.name] = b
211 return b
212
Shawn O. Pearce366ad212009-05-19 12:47:37 -0700213 def GetSubSections(self, section):
214 """List all subsection names matching $section.*.*
215 """
216 return self._sections.get(section, set())
217
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700218 def HasSection(self, section, subsection = ''):
219 """Does at least one key in section.subsection exist?
220 """
221 try:
222 return subsection in self._sections[section]
223 except KeyError:
224 return False
225
Shawn O. Pearce13111b42011-09-19 11:00:31 -0700226 def UrlInsteadOf(self, url):
227 """Resolve any url.*.insteadof references.
228 """
229 for new_url in self.GetSubSections('url'):
Dan Willemsen4e4d40f2013-10-28 22:28:42 -0700230 for old_url in self.GetString('url.%s.insteadof' % new_url, True):
231 if old_url is not None and url.startswith(old_url):
232 return new_url + url[len(old_url):]
Shawn O. Pearce13111b42011-09-19 11:00:31 -0700233 return url
234
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700235 @property
236 def _sections(self):
237 d = self._section_dict
238 if d is None:
239 d = {}
240 for name in self._cache.keys():
241 p = name.split('.')
242 if 2 == len(p):
243 section = p[0]
244 subsect = ''
245 else:
246 section = p[0]
247 subsect = '.'.join(p[1:-1])
248 if section not in d:
249 d[section] = set()
250 d[section].add(subsect)
251 self._section_dict = d
252 return d
253
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700254 @property
255 def _cache(self):
256 if self._cache_dict is None:
257 self._cache_dict = self._Read()
258 return self._cache_dict
259
260 def _Read(self):
Anthony King85b24ac2014-05-06 15:57:48 +0100261 d = self._ReadJson()
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700262 if d is None:
263 d = self._ReadGit()
Anthony King85b24ac2014-05-06 15:57:48 +0100264 self._SaveJson(d)
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700265 return d
266
Anthony King85b24ac2014-05-06 15:57:48 +0100267 def _ReadJson(self):
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700268 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100269 if os.path.getmtime(self._json) \
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700270 <= os.path.getmtime(self.file):
Anthony King85b24ac2014-05-06 15:57:48 +0100271 os.remove(self._json)
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700272 return None
273 except OSError:
274 return None
275 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100276 Trace(': parsing %s', self.file)
277 fd = open(self._json)
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700278 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100279 return json.load(fd)
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700280 finally:
281 fd.close()
Anthony King85b24ac2014-05-06 15:57:48 +0100282 except (IOError, ValueError):
283 os.remove(self._json)
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700284 return None
285
Anthony King85b24ac2014-05-06 15:57:48 +0100286 def _SaveJson(self, cache):
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700287 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100288 fd = open(self._json, 'w')
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700289 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100290 json.dump(cache, fd, indent=2)
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700291 finally:
292 fd.close()
Anthony King85b24ac2014-05-06 15:57:48 +0100293 except (IOError, TypeError):
Anthony Kingb1d1fd72015-06-03 17:02:26 +0100294 if os.path.exists(self._json):
Anthony King85b24ac2014-05-06 15:57:48 +0100295 os.remove(self._json)
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700296
297 def _ReadGit(self):
David Aguilar438c5472009-06-28 15:09:16 -0700298 """
299 Read configuration data from git.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700300
David Aguilar438c5472009-06-28 15:09:16 -0700301 This internal method populates the GitConfig cache.
302
303 """
David Aguilar438c5472009-06-28 15:09:16 -0700304 c = {}
Shawn O. Pearcec24c7202009-07-02 16:12:57 -0700305 d = self._do('--null', '--list')
306 if d is None:
307 return c
Chirayu Desai0eb35cb2013-11-19 18:46:29 +0530308 for line in d.decode('utf-8').rstrip('\0').split('\0'): # pylint: disable=W1401
309 # Backslash is not anomalous
David Aguilar438c5472009-06-28 15:09:16 -0700310 if '\n' in line:
David Pursehousec1b86a22012-11-14 11:36:51 +0900311 key, val = line.split('\n', 1)
David Aguilar438c5472009-06-28 15:09:16 -0700312 else:
David Pursehousec1b86a22012-11-14 11:36:51 +0900313 key = line
314 val = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700315
316 if key in c:
317 c[key].append(val)
318 else:
319 c[key] = [val]
320
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700321 return c
322
323 def _do(self, *args):
324 command = ['config', '--file', self.file]
325 command.extend(args)
326
327 p = GitCommand(None,
328 command,
329 capture_stdout = True,
330 capture_stderr = True)
331 if p.Wait() == 0:
332 return p.stdout
333 else:
334 GitError('git config %s: %s' % (str(args), p.stderr))
335
336
337class RefSpec(object):
338 """A Git refspec line, split into its components:
339
340 forced: True if the line starts with '+'
341 src: Left side of the line
342 dst: Right side of the line
343 """
344
345 @classmethod
346 def FromString(cls, rs):
347 lhs, rhs = rs.split(':', 2)
348 if lhs.startswith('+'):
349 lhs = lhs[1:]
350 forced = True
351 else:
352 forced = False
353 return cls(forced, lhs, rhs)
354
355 def __init__(self, forced, lhs, rhs):
356 self.forced = forced
357 self.src = lhs
358 self.dst = rhs
359
360 def SourceMatches(self, rev):
361 if self.src:
362 if rev == self.src:
363 return True
364 if self.src.endswith('/*') and rev.startswith(self.src[:-1]):
365 return True
366 return False
367
368 def DestMatches(self, ref):
369 if self.dst:
370 if ref == self.dst:
371 return True
372 if self.dst.endswith('/*') and ref.startswith(self.dst[:-1]):
373 return True
374 return False
375
376 def MapSource(self, rev):
377 if self.src.endswith('/*'):
378 return self.dst[:-1] + rev[len(self.src) - 1:]
379 return self.dst
380
381 def __str__(self):
382 s = ''
383 if self.forced:
384 s += '+'
385 if self.src:
386 s += self.src
387 if self.dst:
388 s += ':'
389 s += self.dst
390 return s
391
392
Doug Anderson06d029c2010-10-27 17:06:01 -0700393_master_processes = []
394_master_keys = set()
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700395_ssh_master = True
Doug Anderson0048b692010-12-21 13:39:23 -0800396_master_keys_lock = None
397
398def init_ssh():
399 """Should be called once at the start of repo to init ssh master handling.
400
401 At the moment, all we do is to create our lock.
402 """
403 global _master_keys_lock
404 assert _master_keys_lock is None, "Should only call init_ssh once"
405 _master_keys_lock = _threading.Lock()
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700406
Josh Guilfoyle71985722009-08-16 09:44:40 -0700407def _open_ssh(host, port=None):
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700408 global _ssh_master
409
Doug Anderson0048b692010-12-21 13:39:23 -0800410 # Acquire the lock. This is needed to prevent opening multiple masters for
411 # the same host when we're running "repo sync -jN" (for N > 1) _and_ the
412 # manifest <remote fetch="ssh://xyz"> specifies a different host from the
413 # one that was passed to repo init.
414 _master_keys_lock.acquire()
Doug Anderson06d029c2010-10-27 17:06:01 -0700415 try:
Doug Anderson06d029c2010-10-27 17:06:01 -0700416
Doug Anderson0048b692010-12-21 13:39:23 -0800417 # Check to see whether we already think that the master is running; if we
418 # think it's already running, return right away.
419 if port is not None:
420 key = '%s:%s' % (host, port)
421 else:
422 key = host
423
424 if key in _master_keys:
Doug Anderson06d029c2010-10-27 17:06:01 -0700425 return True
Doug Anderson06d029c2010-10-27 17:06:01 -0700426
Doug Anderson0048b692010-12-21 13:39:23 -0800427 if not _ssh_master \
428 or 'GIT_SSH' in os.environ \
429 or sys.platform in ('win32', 'cygwin'):
430 # failed earlier, or cygwin ssh can't do this
431 #
432 return False
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700433
Doug Anderson0048b692010-12-21 13:39:23 -0800434 # We will make two calls to ssh; this is the common part of both calls.
435 command_base = ['ssh',
436 '-o','ControlPath %s' % ssh_sock(),
437 host]
438 if port is not None:
David Pursehouse8f62fb72012-11-14 12:09:38 +0900439 command_base[1:1] = ['-p', str(port)]
Doug Anderson0048b692010-12-21 13:39:23 -0800440
441 # Since the key wasn't in _master_keys, we think that master isn't running.
442 # ...but before actually starting a master, we'll double-check. This can
443 # be important because we can't tell that that 'git@myhost.com' is the same
444 # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
445 check_command = command_base + ['-O','check']
446 try:
447 Trace(': %s', ' '.join(check_command))
448 check_process = subprocess.Popen(check_command,
449 stdout=subprocess.PIPE,
450 stderr=subprocess.PIPE)
451 check_process.communicate() # read output, but ignore it...
452 isnt_running = check_process.wait()
453
454 if not isnt_running:
455 # Our double-check found that the master _was_ infact running. Add to
456 # the list of keys.
457 _master_keys.add(key)
458 return True
459 except Exception:
460 # Ignore excpetions. We we will fall back to the normal command and print
461 # to the log there.
462 pass
463
464 command = command_base[:1] + \
465 ['-M', '-N'] + \
466 command_base[1:]
467 try:
468 Trace(': %s', ' '.join(command))
469 p = subprocess.Popen(command)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700470 except Exception as e:
Doug Anderson0048b692010-12-21 13:39:23 -0800471 _ssh_master = False
Sarah Owenscecd1d82012-11-01 22:59:27 -0700472 print('\nwarn: cannot enable ssh control master for %s:%s\n%s'
473 % (host,port, str(e)), file=sys.stderr)
Doug Anderson0048b692010-12-21 13:39:23 -0800474 return False
475
Timo Lotterbach05dc46b2016-07-15 16:48:42 +0200476 time.sleep(1)
477 ssh_died = (p.poll() is not None)
478 if ssh_died:
479 return False
480
Doug Anderson0048b692010-12-21 13:39:23 -0800481 _master_processes.append(p)
482 _master_keys.add(key)
Doug Anderson0048b692010-12-21 13:39:23 -0800483 return True
484 finally:
485 _master_keys_lock.release()
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700486
487def close_ssh():
Doug Anderson0048b692010-12-21 13:39:23 -0800488 global _master_keys_lock
489
Shawn O. Pearceca8c32c2010-05-11 18:21:33 -0700490 terminate_ssh_clients()
491
Doug Anderson06d029c2010-10-27 17:06:01 -0700492 for p in _master_processes:
Shawn O. Pearce26120ca2009-06-16 11:49:10 -0700493 try:
494 os.kill(p.pid, SIGTERM)
495 p.wait()
Shawn O. Pearcefb5c8fd2009-06-16 14:57:46 -0700496 except OSError:
Shawn O. Pearce26120ca2009-06-16 11:49:10 -0700497 pass
Doug Anderson06d029c2010-10-27 17:06:01 -0700498 del _master_processes[:]
499 _master_keys.clear()
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700500
Nico Sallembien1c85f4e2010-04-27 14:35:27 -0700501 d = ssh_sock(create=False)
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700502 if d:
503 try:
504 os.rmdir(os.path.dirname(d))
505 except OSError:
506 pass
507
Doug Anderson0048b692010-12-21 13:39:23 -0800508 # We're done with the lock, so we can delete it.
509 _master_keys_lock = None
510
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700511URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
Shawn O. Pearce898e12a2012-03-14 15:22:28 -0700512URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/')
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700513
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -0700514def GetSchemeFromUrl(url):
515 m = URI_ALL.match(url)
516 if m:
517 return m.group(1)
518 return None
519
Dan Willemsen0745bb22015-08-17 13:41:45 -0700520@contextlib.contextmanager
521def GetUrlCookieFile(url, quiet):
522 if url.startswith('persistent-'):
523 try:
524 p = subprocess.Popen(
525 ['git-remote-persistent-https', '-print_config', url],
526 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
527 stderr=subprocess.PIPE)
528 try:
529 cookieprefix = 'http.cookiefile='
530 proxyprefix = 'http.proxy='
531 cookiefile = None
532 proxy = None
533 for line in p.stdout:
534 line = line.strip()
535 if line.startswith(cookieprefix):
536 cookiefile = line[len(cookieprefix):]
537 if line.startswith(proxyprefix):
538 proxy = line[len(proxyprefix):]
539 # Leave subprocess open, as cookie file may be transient.
540 if cookiefile or proxy:
541 yield cookiefile, proxy
542 return
543 finally:
544 p.stdin.close()
545 if p.wait():
546 err_msg = p.stderr.read()
547 if ' -print_config' in err_msg:
548 pass # Persistent proxy doesn't support -print_config.
549 elif not quiet:
550 print(err_msg, file=sys.stderr)
551 except OSError as e:
552 if e.errno == errno.ENOENT:
553 pass # No persistent proxy.
554 raise
555 yield GitConfig.ForUser().GetString('http.cookiefile'), None
556
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700557def _preconnect(url):
558 m = URI_ALL.match(url)
559 if m:
560 scheme = m.group(1)
561 host = m.group(2)
562 if ':' in host:
563 host, port = host.split(':')
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700564 else:
Josh Guilfoyle71985722009-08-16 09:44:40 -0700565 port = None
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700566 if scheme in ('ssh', 'git+ssh', 'ssh+git'):
567 return _open_ssh(host, port)
568 return False
569
570 m = URI_SCP.match(url)
571 if m:
572 host = m.group(1)
Josh Guilfoyle71985722009-08-16 09:44:40 -0700573 return _open_ssh(host)
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700574
Shawn O. Pearce7b4f4352009-06-12 09:06:35 -0700575 return False
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700576
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700577class Remote(object):
578 """Configuration options related to a remote.
579 """
580 def __init__(self, config, name):
581 self._config = config
582 self.name = name
583 self.url = self._Get('url')
Steve Raed6480452016-08-10 15:00:00 -0700584 self.pushUrl = self._Get('pushurl')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700585 self.review = self._Get('review')
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800586 self.projectname = self._Get('projectname')
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530587 self.fetch = list(map(RefSpec.FromString,
588 self._Get('fetch', all_keys=True)))
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800589 self._review_url = None
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800590
Ulrik Sjolinb6ea3bf2010-01-03 18:20:17 +0100591 def _InsteadOf(self):
592 globCfg = GitConfig.ForUser()
593 urlList = globCfg.GetSubSections('url')
594 longest = ""
595 longestUrl = ""
596
597 for url in urlList:
598 key = "url." + url + ".insteadOf"
David Pursehouse8a68ff92012-09-24 12:15:13 +0900599 insteadOfList = globCfg.GetString(key, all_keys=True)
Ulrik Sjolinb6ea3bf2010-01-03 18:20:17 +0100600
601 for insteadOf in insteadOfList:
602 if self.url.startswith(insteadOf) \
603 and len(insteadOf) > len(longest):
604 longest = insteadOf
605 longestUrl = url
606
607 if len(longest) == 0:
608 return self.url
609
610 return self.url.replace(longest, longestUrl, 1)
611
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700612 def PreConnectFetch(self):
Ulrik Sjolinb6ea3bf2010-01-03 18:20:17 +0100613 connectionUrl = self._InsteadOf()
614 return _preconnect(connectionUrl)
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700615
Łukasz Gardońbed59ce2017-08-08 10:18:11 +0200616 def ReviewUrl(self, userEmail, validate_certs):
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800617 if self._review_url is None:
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800618 if self.review is None:
619 return None
620
621 u = self.review
Conley Owens7e12e0a2014-10-23 15:40:00 -0700622 if u.startswith('persistent-'):
623 u = u[len('persistent-'):]
Christian Koestlin2ec2a5d2016-12-05 20:32:45 +0100624 if u.split(':')[0] not in ('http', 'https', 'sso', 'ssh'):
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800625 u = 'http://%s' % u
Shawn O. Pearce13cc3842009-03-25 13:54:54 -0700626 if u.endswith('/Gerrit'):
627 u = u[:len(u) - len('/Gerrit')]
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800628 if u.endswith('/ssh_info'):
629 u = u[:len(u) - len('/ssh_info')]
630 if not u.endswith('/'):
David Pursehouse8a68ff92012-09-24 12:15:13 +0900631 u += '/'
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800632 http_url = u
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800633
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700634 if u in REVIEW_CACHE:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800635 self._review_url = REVIEW_CACHE[u]
Shawn O. Pearce1a68dc52011-10-11 14:12:46 -0700636 elif 'REPO_HOST_PORT_INFO' in os.environ:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800637 host, port = os.environ['REPO_HOST_PORT_INFO'].split()
638 self._review_url = self._SshReviewUrl(userEmail, host, port)
639 REVIEW_CACHE[u] = self._review_url
Christian Koestlin2ec2a5d2016-12-05 20:32:45 +0100640 elif u.startswith('sso:') or u.startswith('ssh:'):
Steve Pucci143d8a72014-01-30 09:45:53 -0800641 self._review_url = u # Assume it's right
642 REVIEW_CACHE[u] = self._review_url
Timo Lotterbacheec726c2016-10-07 10:52:08 +0200643 elif 'REPO_IGNORE_SSH_INFO' in os.environ:
644 self._review_url = http_url
645 REVIEW_CACHE[u] = self._review_url
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700646 else:
647 try:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800648 info_url = u + 'ssh_info'
Łukasz Gardońbed59ce2017-08-08 10:18:11 +0200649 if not validate_certs:
650 context = ssl._create_unverified_context()
651 info = urllib.request.urlopen(info_url, context=context).read()
652 else:
653 info = urllib.request.urlopen(info_url).read()
Conley Owens745a39b2013-06-05 13:16:18 -0700654 if info == 'NOT_AVAILABLE' or '<' in info:
655 # If `info` contains '<', we assume the server gave us some sort
656 # of HTML response back, like maybe a login page.
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700657 #
Conley Owens745a39b2013-06-05 13:16:18 -0700658 # Assume HTTP if SSH is not enabled or ssh_info doesn't look right.
Conley Owens2cd38a02014-02-04 15:32:29 -0800659 self._review_url = http_url
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700660 else:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800661 host, port = info.split()
Dan Willemsen16889ba2016-09-22 16:39:06 +0000662 self._review_url = self._SshReviewUrl(userEmail, host, port)
Sarah Owens1f7627f2012-10-31 09:21:55 -0700663 except urllib.error.HTTPError as e:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800664 raise UploadError('%s: %s' % (self.review, str(e)))
Sarah Owens1f7627f2012-10-31 09:21:55 -0700665 except urllib.error.URLError as e:
Shawn O. Pearcebf1fbb22011-10-11 09:31:58 -0700666 raise UploadError('%s: %s' % (self.review, str(e)))
David Pursehouseecf8f2b2013-05-24 12:12:23 +0900667 except HTTPException as e:
668 raise UploadError('%s: %s' % (self.review, e.__class__.__name__))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800669
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800670 REVIEW_CACHE[u] = self._review_url
671 return self._review_url + self.projectname
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800672
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800673 def _SshReviewUrl(self, userEmail, host, port):
Shawn O. Pearce3575b8f2010-07-15 17:00:14 -0700674 username = self._config.GetString('review.%s.username' % self.review)
675 if username is None:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800676 username = userEmail.split('@')[0]
677 return 'ssh://%s@%s:%s/' % (username, host, port)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700678
679 def ToLocal(self, rev):
680 """Convert a remote revision string to something we have locally.
681 """
Yann Droneaud936183a2013-09-12 10:51:18 +0200682 if self.name == '.' or IsId(rev):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700683 return rev
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700684
685 if not rev.startswith('refs/'):
686 rev = R_HEADS + rev
687
688 for spec in self.fetch:
689 if spec.SourceMatches(rev):
690 return spec.MapSource(rev)
Nasser Grainawi909d58b2014-09-19 12:13:04 -0600691
692 if not rev.startswith(R_HEADS):
693 return rev
694
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700695 raise GitError('remote %s does not have %s' % (self.name, rev))
696
697 def WritesTo(self, ref):
698 """True if the remote stores to the tracking ref.
699 """
700 for spec in self.fetch:
701 if spec.DestMatches(ref):
702 return True
703 return False
704
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800705 def ResetFetch(self, mirror=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700706 """Set the fetch refspec to its default value.
707 """
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800708 if mirror:
709 dst = 'refs/heads/*'
710 else:
711 dst = 'refs/remotes/%s/*' % self.name
712 self.fetch = [RefSpec(True, 'refs/heads/*', dst)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700713
714 def Save(self):
715 """Save this remote to the configuration.
716 """
717 self._Set('url', self.url)
Steve Raed6480452016-08-10 15:00:00 -0700718 if self.pushUrl is not None:
719 self._Set('pushurl', self.pushUrl + '/' + self.projectname)
720 else:
721 self._Set('pushurl', self.pushUrl)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700722 self._Set('review', self.review)
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800723 self._Set('projectname', self.projectname)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530724 self._Set('fetch', list(map(str, self.fetch)))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700725
726 def _Set(self, key, value):
727 key = 'remote.%s.%s' % (self.name, key)
728 return self._config.SetString(key, value)
729
David Pursehouse8a68ff92012-09-24 12:15:13 +0900730 def _Get(self, key, all_keys=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700731 key = 'remote.%s.%s' % (self.name, key)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900732 return self._config.GetString(key, all_keys = all_keys)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700733
734
735class Branch(object):
736 """Configuration options related to a single branch.
737 """
738 def __init__(self, config, name):
739 self._config = config
740 self.name = name
741 self.merge = self._Get('merge')
742
743 r = self._Get('remote')
744 if r:
745 self.remote = self._config.GetRemote(r)
746 else:
747 self.remote = None
748
749 @property
750 def LocalMerge(self):
751 """Convert the merge spec to a local name.
752 """
753 if self.remote and self.merge:
754 return self.remote.ToLocal(self.merge)
755 return None
756
757 def Save(self):
758 """Save this branch back into the configuration.
759 """
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700760 if self._config.HasSection('branch', self.name):
761 if self.remote:
762 self._Set('remote', self.remote.name)
763 else:
764 self._Set('remote', None)
765 self._Set('merge', self.merge)
766
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700767 else:
Chirayu Desai303a82f2014-08-19 22:57:17 +0530768 fd = open(self._config.file, 'a')
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700769 try:
770 fd.write('[branch "%s"]\n' % self.name)
771 if self.remote:
772 fd.write('\tremote = %s\n' % self.remote.name)
773 if self.merge:
774 fd.write('\tmerge = %s\n' % self.merge)
775 finally:
776 fd.close()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700777
778 def _Set(self, key, value):
779 key = 'branch.%s.%s' % (self.name, key)
780 return self._config.SetString(key, value)
781
David Pursehouse8a68ff92012-09-24 12:15:13 +0900782 def _Get(self, key, all_keys=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700783 key = 'branch.%s.%s' % (self.name, key)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900784 return self._config.GetString(key, all_keys = all_keys)