blob: daf85d30b81e4bf48efb888cd8509c24401310ad [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Raman Tenneti993af5e2021-05-12 12:00:31 -070015import collections
Colin Cross23acdd32012-04-21 00:33:54 -070016import itertools
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070017import os
Raman Tenneti080877e2021-03-09 15:19:06 -080018import platform
Conley Owensdb728cd2011-09-26 16:34:01 -070019import re
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import sys
David Pursehouse59bbb582013-05-17 10:49:33 +090021import xml.dom.minidom
Mike Frysingeracf63b22019-06-13 02:24:21 -040022import urllib.parse
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070023
Simran Basib9a1b732015-08-20 12:19:28 -070024import gitc_utils
Miguel Gaio1f207762020-07-17 14:09:13 +020025from git_config import GitConfig, IsId
David Pursehousee00aa6b2012-09-11 14:33:51 +090026from git_refs import R_HEADS, HEAD
Renaud Paquayd5cec5e2016-11-01 11:24:03 -070027import platform_utils
Jack Neus6ea0cae2021-07-20 20:52:33 +000028from project import Annotation, RemoteSpec, Project, MetaProject
Mike Frysinger04122b72019-07-31 23:32:58 -040029from error import (ManifestParseError, ManifestInvalidPathError,
30 ManifestInvalidRevisionError)
Raman Tenneti993af5e2021-05-12 12:00:31 -070031from wrapper import Wrapper
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070032
33MANIFEST_FILE_NAME = 'manifest.xml'
Shawn O. Pearce5cc66792008-10-23 16:19:27 -070034LOCAL_MANIFEST_NAME = 'local_manifest.xml'
David Pursehouse2d5a0df2012-11-13 02:50:36 +090035LOCAL_MANIFESTS_DIR_NAME = 'local_manifests'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070036
Raman Tenneti78f4dd32021-06-07 13:27:37 -070037# Add all projects from local manifest into a group.
38LOCAL_MANIFEST_GROUP_PREFIX = 'local:'
39
Raman Tenneti993af5e2021-05-12 12:00:31 -070040# ContactInfo has the self-registered bug url, supplied by the manifest authors.
41ContactInfo = collections.namedtuple('ContactInfo', 'bugurl')
42
Anthony Kingcb07ba72015-03-28 23:26:04 +000043# urljoin gets confused if the scheme is not known.
Joe Kilner6e310792016-10-27 15:53:53 -070044urllib.parse.uses_relative.extend([
45 'ssh',
46 'git',
47 'persistent-https',
48 'sso',
49 'rpc'])
50urllib.parse.uses_netloc.extend([
51 'ssh',
52 'git',
53 'persistent-https',
54 'sso',
55 'rpc'])
Conley Owensdb728cd2011-09-26 16:34:01 -070056
David Pursehouse819827a2020-02-12 15:20:19 +090057
Mike Frysingerbb8ee7f2020-02-22 05:30:12 -050058def XmlBool(node, attr, default=None):
59 """Determine boolean value of |node|'s |attr|.
60
61 Invalid values will issue a non-fatal warning.
62
63 Args:
64 node: XML node whose attributes we access.
65 attr: The attribute to access.
66 default: If the attribute is not set (value is empty), then use this.
67
68 Returns:
69 True if the attribute is a valid string representing true.
70 False if the attribute is a valid string representing false.
71 |default| otherwise.
72 """
73 value = node.getAttribute(attr)
74 s = value.lower()
75 if s == '':
76 return default
77 elif s in {'yes', 'true', '1'}:
78 return True
79 elif s in {'no', 'false', '0'}:
80 return False
81 else:
82 print('warning: manifest: %s="%s": ignoring invalid XML boolean' %
83 (attr, value), file=sys.stderr)
84 return default
85
86
87def XmlInt(node, attr, default=None):
88 """Determine integer value of |node|'s |attr|.
89
90 Args:
91 node: XML node whose attributes we access.
92 attr: The attribute to access.
93 default: If the attribute is not set (value is empty), then use this.
94
95 Returns:
96 The number if the attribute is a valid number.
97
98 Raises:
99 ManifestParseError: The number is invalid.
100 """
101 value = node.getAttribute(attr)
102 if not value:
103 return default
104
105 try:
106 return int(value)
107 except ValueError:
108 raise ManifestParseError('manifest: invalid %s="%s" integer' %
109 (attr, value))
110
111
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700112class _Default(object):
113 """Project defaults within the manifest."""
114
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700115 revisionExpr = None
Conley Owensb6a16e62013-09-25 15:06:09 -0700116 destBranchExpr = None
Nasser Grainawida403412018-05-04 12:53:29 -0600117 upstreamExpr = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700118 remote = None
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700119 sync_j = 1
Anatol Pomazau79770d22012-04-20 14:41:59 -0700120 sync_c = False
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800121 sync_s = False
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900122 sync_tags = True
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700123
Julien Campergue74879922013-10-09 14:38:46 +0200124 def __eq__(self, other):
Jack Neus5ba21202021-06-09 15:21:25 +0000125 if not isinstance(other, _Default):
126 return False
Julien Campergue74879922013-10-09 14:38:46 +0200127 return self.__dict__ == other.__dict__
128
129 def __ne__(self, other):
Jack Neus5ba21202021-06-09 15:21:25 +0000130 if not isinstance(other, _Default):
131 return True
Julien Campergue74879922013-10-09 14:38:46 +0200132 return self.__dict__ != other.__dict__
133
David Pursehouse819827a2020-02-12 15:20:19 +0900134
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700135class _XmlRemote(object):
136 def __init__(self,
137 name,
Yestin Sunb292b982012-07-02 07:32:50 -0700138 alias=None,
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700139 fetch=None,
Steve Raed6480452016-08-10 15:00:00 -0700140 pushUrl=None,
Conley Owensdb728cd2011-09-26 16:34:01 -0700141 manifestUrl=None,
Anthony King36ea2fb2014-05-06 11:54:01 +0100142 review=None,
Jonathan Nieder93719792015-03-17 11:29:58 -0700143 revision=None):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700144 self.name = name
145 self.fetchUrl = fetch
Steve Raed6480452016-08-10 15:00:00 -0700146 self.pushUrl = pushUrl
Conley Owensdb728cd2011-09-26 16:34:01 -0700147 self.manifestUrl = manifestUrl
Yestin Sunb292b982012-07-02 07:32:50 -0700148 self.remoteAlias = alias
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700149 self.reviewUrl = review
Anthony King36ea2fb2014-05-06 11:54:01 +0100150 self.revision = revision
Conley Owensceea3682011-10-20 10:45:47 -0700151 self.resolvedFetchUrl = self._resolveFetchUrl()
Jack Neus6ea0cae2021-07-20 20:52:33 +0000152 self.annotations = []
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700153
David Pursehouse717ece92012-11-13 08:49:16 +0900154 def __eq__(self, other):
Jack Neus5ba21202021-06-09 15:21:25 +0000155 if not isinstance(other, _XmlRemote):
156 return False
Jack Neus6ea0cae2021-07-20 20:52:33 +0000157 return (sorted(self.annotations) == sorted(other.annotations) and
158 self.name == other.name and self.fetchUrl == other.fetchUrl and
159 self.pushUrl == other.pushUrl and self.remoteAlias == other.remoteAlias
160 and self.reviewUrl == other.reviewUrl and self.revision == other.revision)
David Pursehouse717ece92012-11-13 08:49:16 +0900161
162 def __ne__(self, other):
Jack Neus6ea0cae2021-07-20 20:52:33 +0000163 return not self.__eq__(other)
David Pursehouse717ece92012-11-13 08:49:16 +0900164
Conley Owensceea3682011-10-20 10:45:47 -0700165 def _resolveFetchUrl(self):
Jack Neus5ba21202021-06-09 15:21:25 +0000166 if self.fetchUrl is None:
167 return ''
Conley Owensceea3682011-10-20 10:45:47 -0700168 url = self.fetchUrl.rstrip('/')
Conley Owensdb728cd2011-09-26 16:34:01 -0700169 manifestUrl = self.manifestUrl.rstrip('/')
Conley Owens2d0f5082014-01-31 15:03:51 -0800170 # urljoin will gets confused over quite a few things. The ones we care
171 # about here are:
172 # * no scheme in the base url, like <hostname:port>
Anthony Kingcb07ba72015-03-28 23:26:04 +0000173 # We handle no scheme by replacing it with an obscure protocol, gopher
174 # and then replacing it with the original when we are done.
175
Conley Owensdb728cd2011-09-26 16:34:01 -0700176 if manifestUrl.find(':') != manifestUrl.find('/') - 1:
Conley Owens4ccad752015-04-29 10:45:37 -0700177 url = urllib.parse.urljoin('gopher://' + manifestUrl, url)
178 url = re.sub(r'^gopher://', '', url)
Anthony Kingcb07ba72015-03-28 23:26:04 +0000179 else:
180 url = urllib.parse.urljoin(manifestUrl, url)
Shawn Pearcea9f11b32013-01-02 15:40:48 -0800181 return url
Conley Owensceea3682011-10-20 10:45:47 -0700182
183 def ToRemoteSpec(self, projectName):
David Rileye0684ad2017-04-05 00:02:59 -0700184 fetchUrl = self.resolvedFetchUrl.rstrip('/')
185 url = fetchUrl + '/' + projectName
Yestin Sunb292b982012-07-02 07:32:50 -0700186 remoteName = self.name
Conley Owens1e7ab2a2013-10-08 17:26:57 -0700187 if self.remoteAlias:
David Pursehouse37128b62013-10-15 10:48:40 +0900188 remoteName = self.remoteAlias
Dan Willemsen96c2d652016-04-06 16:03:54 -0700189 return RemoteSpec(remoteName,
190 url=url,
Steve Raed6480452016-08-10 15:00:00 -0700191 pushUrl=self.pushUrl,
Dan Willemsen96c2d652016-04-06 16:03:54 -0700192 review=self.reviewUrl,
David Rileye0684ad2017-04-05 00:02:59 -0700193 orig_name=self.name,
194 fetchUrl=self.fetchUrl)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700195
Jack Neus6ea0cae2021-07-20 20:52:33 +0000196 def AddAnnotation(self, name, value, keep):
197 self.annotations.append(Annotation(name, value, keep))
198
David Pursehouse819827a2020-02-12 15:20:19 +0900199
Shawn O. Pearcec8a300f2009-05-18 13:19:57 -0700200class XmlManifest(object):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700201 """manages the repo configuration file"""
202
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -0400203 def __init__(self, repodir, manifest_file, local_manifests=None):
204 """Initialize.
205
206 Args:
207 repodir: Path to the .repo/ dir for holding all internal checkout state.
208 It must be in the top directory of the repo client checkout.
209 manifest_file: Full path to the manifest file to parse. This will usually
210 be |repodir|/|MANIFEST_FILE_NAME|.
211 local_manifests: Full path to the directory of local override manifests.
212 This will usually be |repodir|/|LOCAL_MANIFESTS_DIR_NAME|.
213 """
214 # TODO(vapier): Move this out of this class.
215 self.globalConfig = GitConfig.ForUser()
216
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700217 self.repodir = os.path.abspath(repodir)
218 self.topdir = os.path.dirname(self.repodir)
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -0400219 self.manifestFile = manifest_file
220 self.local_manifests = local_manifests
Basil Gelloc7453502018-05-25 20:23:52 +0300221 self._load_local_manifests = True
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700222
223 self.repoProject = MetaProject(self, 'repo',
David Pursehouseabdf7502020-02-12 14:58:39 +0900224 gitdir=os.path.join(repodir, 'repo/.git'),
225 worktree=os.path.join(repodir, 'repo'))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700226
Mike Frysinger979d5bd2020-02-09 02:28:34 -0500227 mp = MetaProject(self, 'manifests',
228 gitdir=os.path.join(repodir, 'manifests.git'),
229 worktree=os.path.join(repodir, 'manifests'))
230 self.manifestProject = mp
231
232 # This is a bit hacky, but we're in a chicken & egg situation: all the
233 # normal repo settings live in the manifestProject which we just setup
234 # above, so we couldn't easily query before that. We assume Project()
235 # init doesn't care if this changes afterwards.
Mike Frysingerd957ec62020-02-24 14:40:25 -0500236 if os.path.exists(mp.gitdir) and mp.config.GetBoolean('repo.worktree'):
Mike Frysinger979d5bd2020-02-09 02:28:34 -0500237 mp.use_git_worktrees = True
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700238
239 self._Unload()
240
Basil Gelloc7453502018-05-25 20:23:52 +0300241 def Override(self, name, load_local_manifests=True):
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700242 """Use a different manifest, just for the current instantiation.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700243 """
Basil Gelloc7453502018-05-25 20:23:52 +0300244 path = None
245
246 # Look for a manifest by path in the filesystem (including the cwd).
247 if not load_local_manifests:
248 local_path = os.path.abspath(name)
249 if os.path.isfile(local_path):
250 path = local_path
251
252 # Look for manifests by name from the manifests repo.
253 if path is None:
254 path = os.path.join(self.manifestProject.worktree, name)
255 if not os.path.isfile(path):
256 raise ManifestParseError('manifest %s not found' % name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700257
258 old = self.manifestFile
259 try:
Basil Gelloc7453502018-05-25 20:23:52 +0300260 self._load_local_manifests = load_local_manifests
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700261 self.manifestFile = path
262 self._Unload()
263 self._Load()
264 finally:
265 self.manifestFile = old
266
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700267 def Link(self, name):
268 """Update the repo metadata to use a different manifest.
269 """
270 self.Override(name)
271
Mike Frysingera269b1c2020-02-21 00:49:41 -0500272 # Old versions of repo would generate symlinks we need to clean up.
Mike Frysinger9d96f582021-09-28 11:27:24 -0400273 platform_utils.remove(self.manifestFile, missing_ok=True)
Mike Frysingera269b1c2020-02-21 00:49:41 -0500274 # This file is interpreted as if it existed inside the manifest repo.
275 # That allows us to use <include> with the relative file name.
276 with open(self.manifestFile, 'w') as fp:
277 fp.write("""<?xml version="1.0" encoding="UTF-8"?>
278<!--
279DO NOT EDIT THIS FILE! It is generated by repo and changes will be discarded.
280If you want to use a different manifest, use `repo init -m <file>` instead.
281
282If you want to customize your checkout by overriding manifest settings, use
283the local_manifests/ directory instead.
284
285For more information on repo manifests, check out:
286https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
287-->
288<manifest>
289 <include name="%s" />
290</manifest>
291""" % (name,))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700292
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800293 def _RemoteToXml(self, r, doc, root):
294 e = doc.createElement('remote')
295 root.appendChild(e)
296 e.setAttribute('name', r.name)
297 e.setAttribute('fetch', r.fetchUrl)
Steve Raed6480452016-08-10 15:00:00 -0700298 if r.pushUrl is not None:
299 e.setAttribute('pushurl', r.pushUrl)
Conley Owens1e7ab2a2013-10-08 17:26:57 -0700300 if r.remoteAlias is not None:
301 e.setAttribute('alias', r.remoteAlias)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800302 if r.reviewUrl is not None:
303 e.setAttribute('review', r.reviewUrl)
Anthony King36ea2fb2014-05-06 11:54:01 +0100304 if r.revision is not None:
305 e.setAttribute('revision', r.revision)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800306
Jack Neus6ea0cae2021-07-20 20:52:33 +0000307 for a in r.annotations:
308 if a.keep == 'true':
309 ae = doc.createElement('annotation')
310 ae.setAttribute('name', a.name)
311 ae.setAttribute('value', a.value)
312 e.appendChild(ae)
313
Mike Frysinger51e39d52020-12-04 05:32:06 -0500314 def _ParseList(self, field):
315 """Parse fields that contain flattened lists.
316
317 These are whitespace & comma separated. Empty elements will be discarded.
318 """
319 return [x for x in re.split(r'[,\s]+', field) if x]
Josh Triplett884a3872014-06-12 14:57:29 -0700320
Mike Frysinger23411d32020-09-02 04:31:10 -0400321 def ToXml(self, peg_rev=False, peg_rev_upstream=True, peg_rev_dest_branch=True, groups=None):
322 """Return the current manifest XML."""
Colin Cross5acde752012-03-28 20:15:45 -0700323 mp = self.manifestProject
324
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700325 if groups is None:
326 groups = mp.config.GetString('manifest.groups')
Matt Gumbel0c635bb2012-12-21 10:14:53 -0800327 if groups:
Mike Frysinger51e39d52020-12-04 05:32:06 -0500328 groups = self._ParseList(groups)
Colin Cross5acde752012-03-28 20:15:45 -0700329
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800330 doc = xml.dom.minidom.Document()
331 root = doc.createElement('manifest')
332 doc.appendChild(root)
333
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700334 # Save out the notice. There's a little bit of work here to give it the
335 # right whitespace, which assumes that the notice is automatically indented
336 # by 4 by minidom.
337 if self.notice:
338 notice_element = root.appendChild(doc.createElement('notice'))
339 notice_lines = self.notice.splitlines()
David Pursehouse54a4e602020-02-12 14:31:05 +0900340 indented_notice = ('\n'.join(" " * 4 + line for line in notice_lines))[4:]
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700341 notice_element.appendChild(doc.createTextNode(indented_notice))
342
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800343 d = self.default
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800344
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530345 for r in sorted(self.remotes):
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800346 self._RemoteToXml(self.remotes[r], doc, root)
347 if self.remotes:
348 root.appendChild(doc.createTextNode(''))
349
350 have_default = False
351 e = doc.createElement('default')
352 if d.remote:
353 have_default = True
354 e.setAttribute('remote', d.remote.name)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700355 if d.revisionExpr:
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800356 have_default = True
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700357 e.setAttribute('revision', d.revisionExpr)
Simon Ruggier7e59de22015-07-24 12:50:06 +0200358 if d.destBranchExpr:
359 have_default = True
360 e.setAttribute('dest-branch', d.destBranchExpr)
Nasser Grainawida403412018-05-04 12:53:29 -0600361 if d.upstreamExpr:
362 have_default = True
363 e.setAttribute('upstream', d.upstreamExpr)
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700364 if d.sync_j > 1:
365 have_default = True
366 e.setAttribute('sync-j', '%d' % d.sync_j)
Anatol Pomazau79770d22012-04-20 14:41:59 -0700367 if d.sync_c:
368 have_default = True
369 e.setAttribute('sync-c', 'true')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800370 if d.sync_s:
371 have_default = True
372 e.setAttribute('sync-s', 'true')
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900373 if not d.sync_tags:
374 have_default = True
375 e.setAttribute('sync-tags', 'false')
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800376 if have_default:
377 root.appendChild(e)
378 root.appendChild(doc.createTextNode(''))
379
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700380 if self._manifest_server:
381 e = doc.createElement('manifest-server')
382 e.setAttribute('url', self._manifest_server)
383 root.appendChild(e)
384 root.appendChild(doc.createTextNode(''))
385
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800386 def output_projects(parent, parent_node, projects):
David James8d201162013-10-11 17:03:19 -0700387 for project_name in projects:
388 for project in self._projects[project_name]:
389 output_project(parent, parent_node, project)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800390
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800391 def output_project(parent, parent_node, p):
Colin Cross5acde752012-03-28 20:15:45 -0700392 if not p.MatchesGroups(groups):
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800393 return
394
395 name = p.name
396 relpath = p.relpath
397 if parent:
398 name = self._UnjoinName(parent.name, name)
399 relpath = self._UnjoinRelpath(parent.relpath, relpath)
Colin Cross5acde752012-03-28 20:15:45 -0700400
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800401 e = doc.createElement('project')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800402 parent_node.appendChild(e)
403 e.setAttribute('name', name)
404 if relpath != name:
405 e.setAttribute('path', relpath)
Conley Owensa17d7af2013-10-16 14:38:09 -0700406 remoteName = None
407 if d.remote:
Dan Willemsen96c2d652016-04-06 16:03:54 -0700408 remoteName = d.remote.name
409 if not d.remote or p.remote.orig_name != remoteName:
410 remoteName = p.remote.orig_name
Anthony King36ea2fb2014-05-06 11:54:01 +0100411 e.setAttribute('remote', remoteName)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800412 if peg_rev:
413 if self.IsMirror:
Brian Harring14a66742012-09-28 20:21:57 -0700414 value = p.bare_git.rev_parse(p.revisionExpr + '^0')
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800415 else:
Brian Harring14a66742012-09-28 20:21:57 -0700416 value = p.work_git.rev_parse(HEAD + '^0')
417 e.setAttribute('revision', value)
Conley Owens551dfec2015-07-10 14:54:54 -0700418 if peg_rev_upstream:
419 if p.upstream:
420 e.setAttribute('upstream', p.upstream)
421 elif value != p.revisionExpr:
422 # Only save the origin if the origin is not a sha1, and the default
423 # isn't our value
424 e.setAttribute('upstream', p.revisionExpr)
Sean McAllisteraf908cb2020-04-20 08:41:58 -0600425
426 if peg_rev_dest_branch:
427 if p.dest_branch:
428 e.setAttribute('dest-branch', p.dest_branch)
429 elif value != p.revisionExpr:
430 e.setAttribute('dest-branch', p.revisionExpr)
431
Anthony King36ea2fb2014-05-06 11:54:01 +0100432 else:
Dan Willemsen96c2d652016-04-06 16:03:54 -0700433 revision = self.remotes[p.remote.orig_name].revision or d.revisionExpr
Anthony King36ea2fb2014-05-06 11:54:01 +0100434 if not revision or revision != p.revisionExpr:
435 e.setAttribute('revision', p.revisionExpr)
Raman Tennetib5c5a5e2021-02-06 09:44:15 -0800436 elif p.revisionId:
437 e.setAttribute('revision', p.revisionId)
Nasser Grainawida403412018-05-04 12:53:29 -0600438 if (p.upstream and (p.upstream != p.revisionExpr or
439 p.upstream != d.upstreamExpr)):
Mani Chandel7a91d512014-07-24 16:27:08 +0530440 e.setAttribute('upstream', p.upstream)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800441
Simon Ruggier7e59de22015-07-24 12:50:06 +0200442 if p.dest_branch and p.dest_branch != d.destBranchExpr:
443 e.setAttribute('dest-branch', p.dest_branch)
444
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800445 for c in p.copyfiles:
446 ce = doc.createElement('copyfile')
447 ce.setAttribute('src', c.src)
448 ce.setAttribute('dest', c.dest)
449 e.appendChild(ce)
450
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500451 for l in p.linkfiles:
452 le = doc.createElement('linkfile')
453 le.setAttribute('src', l.src)
454 le.setAttribute('dest', l.dest)
455 e.appendChild(le)
456
Conley Owensbb1b5f52012-08-13 13:11:18 -0700457 default_groups = ['all', 'name:%s' % p.name, 'path:%s' % p.relpath]
Dmitry Fink17f85ea2012-08-06 14:52:29 -0700458 egroups = [g for g in p.groups if g not in default_groups]
Conley Owens971de8e2012-04-16 10:36:08 -0700459 if egroups:
460 e.setAttribute('groups', ','.join(egroups))
Colin Cross5acde752012-03-28 20:15:45 -0700461
James W. Mills24c13082012-04-12 15:04:13 -0500462 for a in p.annotations:
463 if a.keep == "true":
464 ae = doc.createElement('annotation')
465 ae.setAttribute('name', a.name)
466 ae.setAttribute('value', a.value)
467 e.appendChild(ae)
468
Anatol Pomazau79770d22012-04-20 14:41:59 -0700469 if p.sync_c:
470 e.setAttribute('sync-c', 'true')
471
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800472 if p.sync_s:
473 e.setAttribute('sync-s', 'true')
474
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900475 if not p.sync_tags:
476 e.setAttribute('sync-tags', 'false')
477
Dan Willemsen88409222015-08-17 15:29:10 -0700478 if p.clone_depth:
479 e.setAttribute('clone-depth', str(p.clone_depth))
480
Simran Basib9a1b732015-08-20 12:19:28 -0700481 self._output_manifest_project_extras(p, e)
482
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800483 if p.subprojects:
David James8d201162013-10-11 17:03:19 -0700484 subprojects = set(subp.name for subp in p.subprojects)
485 output_projects(p, e, list(sorted(subprojects)))
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800486
David James8d201162013-10-11 17:03:19 -0700487 projects = set(p.name for p in self._paths.values() if not p.parent)
488 output_projects(None, root, list(sorted(projects)))
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800489
Doug Anderson37282b42011-03-04 11:54:18 -0800490 if self._repo_hooks_project:
491 root.appendChild(doc.createTextNode(''))
492 e = doc.createElement('repo-hooks')
493 e.setAttribute('in-project', self._repo_hooks_project.name)
494 e.setAttribute('enabled-list',
495 ' '.join(self._repo_hooks_project.enabled_repo_hooks))
496 root.appendChild(e)
497
Raman Tenneti1bb4fb22021-01-07 16:50:45 -0800498 if self._superproject:
499 root.appendChild(doc.createTextNode(''))
500 e = doc.createElement('superproject')
501 e.setAttribute('name', self._superproject['name'])
502 remoteName = None
503 if d.remote:
504 remoteName = d.remote.name
505 remote = self._superproject.get('remote')
506 if not d.remote or remote.orig_name != remoteName:
507 remoteName = remote.orig_name
508 e.setAttribute('remote', remoteName)
Xin Lie0b16a22021-09-26 23:20:32 -0700509 revision = remote.revision or d.revisionExpr
510 if not revision or revision != self._superproject['revision']:
511 e.setAttribute('revision', self._superproject['revision'])
Raman Tenneti1bb4fb22021-01-07 16:50:45 -0800512 root.appendChild(e)
513
Raman Tenneti993af5e2021-05-12 12:00:31 -0700514 if self._contactinfo.bugurl != Wrapper().BUG_URL:
Raman Tenneti1c3f57e2021-05-04 12:32:13 -0700515 root.appendChild(doc.createTextNode(''))
516 e = doc.createElement('contactinfo')
Raman Tenneti993af5e2021-05-12 12:00:31 -0700517 e.setAttribute('bugurl', self._contactinfo.bugurl)
Raman Tenneti1c3f57e2021-05-04 12:32:13 -0700518 root.appendChild(e)
519
Mike Frysinger23411d32020-09-02 04:31:10 -0400520 return doc
521
522 def ToDict(self, **kwargs):
523 """Return the current manifest as a dictionary."""
524 # Elements that may only appear once.
525 SINGLE_ELEMENTS = {
526 'notice',
527 'default',
528 'manifest-server',
529 'repo-hooks',
Raman Tenneti1bb4fb22021-01-07 16:50:45 -0800530 'superproject',
Raman Tenneti1c3f57e2021-05-04 12:32:13 -0700531 'contactinfo',
Mike Frysinger23411d32020-09-02 04:31:10 -0400532 }
533 # Elements that may be repeated.
534 MULTI_ELEMENTS = {
535 'remote',
536 'remove-project',
537 'project',
538 'extend-project',
539 'include',
540 # These are children of 'project' nodes.
541 'annotation',
542 'project',
543 'copyfile',
544 'linkfile',
545 }
546
547 doc = self.ToXml(**kwargs)
548 ret = {}
549
550 def append_children(ret, node):
551 for child in node.childNodes:
552 if child.nodeType == xml.dom.Node.ELEMENT_NODE:
553 attrs = child.attributes
554 element = dict((attrs.item(i).localName, attrs.item(i).value)
555 for i in range(attrs.length))
556 if child.nodeName in SINGLE_ELEMENTS:
557 ret[child.nodeName] = element
558 elif child.nodeName in MULTI_ELEMENTS:
559 ret.setdefault(child.nodeName, []).append(element)
560 else:
561 raise ManifestParseError('Unhandled element "%s"' % (child.nodeName,))
562
563 append_children(element, child)
564
565 append_children(ret, doc.firstChild)
566
567 return ret
568
569 def Save(self, fd, **kwargs):
570 """Write the current manifest out to the given file descriptor."""
571 doc = self.ToXml(**kwargs)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800572 doc.writexml(fd, '', ' ', '\n', 'UTF-8')
573
Simran Basib9a1b732015-08-20 12:19:28 -0700574 def _output_manifest_project_extras(self, p, e):
575 """Manifests can modify e if they support extra project attributes."""
Simran Basib9a1b732015-08-20 12:19:28 -0700576
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700577 @property
David James8d201162013-10-11 17:03:19 -0700578 def paths(self):
579 self._Load()
580 return self._paths
581
582 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700583 def projects(self):
584 self._Load()
Anthony Kingd58bfe52014-05-05 23:30:49 +0100585 return list(self._paths.values())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700586
587 @property
588 def remotes(self):
589 self._Load()
590 return self._remotes
591
592 @property
593 def default(self):
594 self._Load()
595 return self._default
596
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800597 @property
Doug Anderson37282b42011-03-04 11:54:18 -0800598 def repo_hooks_project(self):
599 self._Load()
600 return self._repo_hooks_project
601
602 @property
Raman Tenneti1bb4fb22021-01-07 16:50:45 -0800603 def superproject(self):
604 self._Load()
605 return self._superproject
606
607 @property
Raman Tenneti1c3f57e2021-05-04 12:32:13 -0700608 def contactinfo(self):
609 self._Load()
610 return self._contactinfo
611
612 @property
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700613 def notice(self):
614 self._Load()
615 return self._notice
616
617 @property
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700618 def manifest_server(self):
619 self._Load()
Shawn O. Pearce34fb20f2011-11-30 13:41:02 -0800620 return self._manifest_server
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700621
622 @property
Xin Lid79a4bc2020-05-20 16:03:45 -0700623 def CloneBundle(self):
624 clone_bundle = self.manifestProject.config.GetBoolean('repo.clonebundle')
625 if clone_bundle is None:
626 return False if self.manifestProject.config.GetBoolean('repo.partialclone') else True
627 else:
628 return clone_bundle
629
630 @property
Xin Li745be2e2019-06-03 11:24:30 -0700631 def CloneFilter(self):
632 if self.manifestProject.config.GetBoolean('repo.partialclone'):
633 return self.manifestProject.config.GetString('repo.clonefilter')
634 return None
635
636 @property
Raman Tennetif32f2432021-04-12 20:57:25 -0700637 def PartialCloneExclude(self):
638 exclude = self.manifest.manifestProject.config.GetString(
639 'repo.partialcloneexclude') or ''
640 return set(x.strip() for x in exclude.split(','))
641
642 @property
Michael Kellyc34b91c2021-07-02 09:25:48 -0700643 def UseLocalManifests(self):
644 return self._load_local_manifests
645
646 def SetUseLocalManifests(self, value):
647 self._load_local_manifests = value
648
649 @property
Raman Tennetifeb28912021-05-02 19:47:29 -0700650 def HasLocalManifests(self):
651 return self._load_local_manifests and self.local_manifests
652
653 @property
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800654 def IsMirror(self):
655 return self.manifestProject.config.GetBoolean('repo.mirror')
656
Julien Campergue335f5ef2013-10-16 11:02:35 +0200657 @property
Mike Frysinger979d5bd2020-02-09 02:28:34 -0500658 def UseGitWorktrees(self):
659 return self.manifestProject.config.GetBoolean('repo.worktree')
660
661 @property
Julien Campergue335f5ef2013-10-16 11:02:35 +0200662 def IsArchive(self):
663 return self.manifestProject.config.GetBoolean('repo.archive')
664
Martin Kellye4e94d22017-03-21 16:05:12 -0700665 @property
666 def HasSubmodules(self):
667 return self.manifestProject.config.GetBoolean('repo.submodules')
668
XD Trol630876f2022-01-17 23:29:04 +0800669 @property
670 def EnableGitLfs(self):
671 return self.manifestProject.config.GetBoolean('repo.git-lfs')
672
Raman Tenneti080877e2021-03-09 15:19:06 -0800673 def GetDefaultGroupsStr(self):
674 """Returns the default group string for the platform."""
675 return 'default,platform-' + platform.system().lower()
676
677 def GetGroupsStr(self):
678 """Returns the manifest group string that should be synced."""
679 groups = self.manifestProject.config.GetString('manifest.groups')
680 if not groups:
681 groups = self.GetDefaultGroupsStr()
682 return groups
683
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700684 def _Unload(self):
685 self._loaded = False
686 self._projects = {}
David James8d201162013-10-11 17:03:19 -0700687 self._paths = {}
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700688 self._remotes = {}
689 self._default = None
Doug Anderson37282b42011-03-04 11:54:18 -0800690 self._repo_hooks_project = None
Raman Tenneti1bb4fb22021-01-07 16:50:45 -0800691 self._superproject = {}
Raman Tenneti993af5e2021-05-12 12:00:31 -0700692 self._contactinfo = ContactInfo(Wrapper().BUG_URL)
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700693 self._notice = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700694 self.branch = None
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700695 self._manifest_server = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700696
697 def _Load(self):
698 if not self._loaded:
Shawn O. Pearce2450a292008-11-04 08:22:07 -0800699 m = self.manifestProject
700 b = m.GetBranch(m.CurrentBranch).merge
Shawn O. Pearce21c5c342009-06-25 16:47:30 -0700701 if b is not None and b.startswith(R_HEADS):
Shawn O. Pearce2450a292008-11-04 08:22:07 -0800702 b = b[len(R_HEADS):]
703 self.branch = b
704
Mike Frysinger54133972021-03-01 21:38:08 -0500705 # The manifestFile was specified by the user which is why we allow include
706 # paths to point anywhere.
Colin Cross23acdd32012-04-21 00:33:54 -0700707 nodes = []
Mike Frysinger54133972021-03-01 21:38:08 -0500708 nodes.append(self._ParseManifestXml(
709 self.manifestFile, self.manifestProject.worktree,
710 restrict_includes=False))
Shawn O. Pearce5cc66792008-10-23 16:19:27 -0700711
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -0400712 if self._load_local_manifests and self.local_manifests:
Basil Gelloc7453502018-05-25 20:23:52 +0300713 try:
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -0400714 for local_file in sorted(platform_utils.listdir(self.local_manifests)):
Basil Gelloc7453502018-05-25 20:23:52 +0300715 if local_file.endswith('.xml'):
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -0400716 local = os.path.join(self.local_manifests, local_file)
Mike Frysinger54133972021-03-01 21:38:08 -0500717 # Since local manifests are entirely managed by the user, allow
718 # them to point anywhere the user wants.
719 nodes.append(self._ParseManifestXml(
Raman Tenneti78f4dd32021-06-07 13:27:37 -0700720 local, self.repodir,
721 parent_groups=f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}',
722 restrict_includes=False))
Basil Gelloc7453502018-05-25 20:23:52 +0300723 except OSError:
724 pass
David Pursehouse2d5a0df2012-11-13 02:50:36 +0900725
Joe Onorato26e24752013-01-11 12:35:53 -0800726 try:
727 self._ParseManifest(nodes)
728 except ManifestParseError as e:
729 # There was a problem parsing, unload ourselves in case they catch
730 # this error and try again later, we will show the correct error
731 self._Unload()
732 raise e
Shawn O. Pearce5cc66792008-10-23 16:19:27 -0700733
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800734 if self.IsMirror:
735 self._AddMetaProjectMirror(self.repoProject)
736 self._AddMetaProjectMirror(self.manifestProject)
737
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700738 self._loaded = True
739
Mike Frysinger54133972021-03-01 21:38:08 -0500740 def _ParseManifestXml(self, path, include_root, parent_groups='',
741 restrict_includes=True):
742 """Parse a manifest XML and return the computed nodes.
743
744 Args:
745 path: The XML file to read & parse.
746 include_root: The path to interpret include "name"s relative to.
747 parent_groups: The groups to apply to this projects.
748 restrict_includes: Whether to constrain the "name" attribute of includes.
749
750 Returns:
751 List of XML nodes.
752 """
David Pursehousef7fc8a92012-11-13 04:00:28 +0900753 try:
754 root = xml.dom.minidom.parse(path)
David Pursehouse2d5a0df2012-11-13 02:50:36 +0900755 except (OSError, xml.parsers.expat.ExpatError) as e:
David Pursehousef7fc8a92012-11-13 04:00:28 +0900756 raise ManifestParseError("error parsing manifest %s: %s" % (path, e))
757
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700758 if not root or not root.childNodes:
Brian Harring26448742011-04-28 05:04:41 -0700759 raise ManifestParseError("no root node in %s" % (path,))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700760
Jooncheol Park34acdd22012-08-27 02:25:59 +0900761 for manifest in root.childNodes:
762 if manifest.nodeName == 'manifest':
763 break
764 else:
Brian Harring26448742011-04-28 05:04:41 -0700765 raise ManifestParseError("no <manifest> in %s" % (path,))
766
Colin Cross23acdd32012-04-21 00:33:54 -0700767 nodes = []
David Pursehouse65b0ba52018-06-24 16:21:51 +0900768 for node in manifest.childNodes:
David Pursehousec1b86a22012-11-14 11:36:51 +0900769 if node.nodeName == 'include':
770 name = self._reqatt(node, 'name')
Mike Frysinger54133972021-03-01 21:38:08 -0500771 if restrict_includes:
772 msg = self._CheckLocalPath(name)
773 if msg:
774 raise ManifestInvalidPathError(
775 '<include> invalid "name": %s: %s' % (name, msg))
Fredrik de Groot352c93b2020-10-06 12:55:14 +0200776 include_groups = ''
777 if parent_groups:
778 include_groups = parent_groups
779 if node.hasAttribute('groups'):
780 include_groups = node.getAttribute('groups') + ',' + include_groups
David Pursehousec1b86a22012-11-14 11:36:51 +0900781 fp = os.path.join(include_root, name)
782 if not os.path.isfile(fp):
Mike Frysinger54133972021-03-01 21:38:08 -0500783 raise ManifestParseError("include [%s/]%s doesn't exist or isn't a file"
784 % (include_root, name))
David Pursehousec1b86a22012-11-14 11:36:51 +0900785 try:
Fredrik de Groot352c93b2020-10-06 12:55:14 +0200786 nodes.extend(self._ParseManifestXml(fp, include_root, include_groups))
David Pursehousec1b86a22012-11-14 11:36:51 +0900787 # should isolate this to the exact exception, but that's
788 # tricky. actual parsing implementation may vary.
Mike Frysinger54133972021-03-01 21:38:08 -0500789 except (KeyboardInterrupt, RuntimeError, SystemExit, ManifestParseError):
David Pursehousec1b86a22012-11-14 11:36:51 +0900790 raise
791 except Exception as e:
792 raise ManifestParseError(
Mike Frysingerec558df2019-07-05 01:38:05 -0400793 "failed parsing included manifest %s: %s" % (name, e))
David Pursehousec1b86a22012-11-14 11:36:51 +0900794 else:
Fredrik de Groot352c93b2020-10-06 12:55:14 +0200795 if parent_groups and node.nodeName == 'project':
796 nodeGroups = parent_groups
797 if node.hasAttribute('groups'):
798 nodeGroups = node.getAttribute('groups') + ',' + nodeGroups
799 node.setAttribute('groups', nodeGroups)
David Pursehousec1b86a22012-11-14 11:36:51 +0900800 nodes.append(node)
Colin Cross23acdd32012-04-21 00:33:54 -0700801 return nodes
Brian Harring26448742011-04-28 05:04:41 -0700802
Colin Cross23acdd32012-04-21 00:33:54 -0700803 def _ParseManifest(self, node_list):
804 for node in itertools.chain(*node_list):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700805 if node.nodeName == 'remote':
806 remote = self._ParseRemote(node)
David Pursehouse717ece92012-11-13 08:49:16 +0900807 if remote:
808 if remote.name in self._remotes:
809 if remote != self._remotes[remote.name]:
810 raise ManifestParseError(
811 'remote %s already exists with different attributes' %
812 (remote.name))
813 else:
814 self._remotes[remote.name] = remote
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700815
Colin Cross23acdd32012-04-21 00:33:54 -0700816 for node in itertools.chain(*node_list):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700817 if node.nodeName == 'default':
Julien Campergue74879922013-10-09 14:38:46 +0200818 new_default = self._ParseDefault(node)
Jack Neusb8c84482021-06-15 14:28:30 +0000819 emptyDefault = not node.hasAttributes() and not node.hasChildNodes()
Julien Campergue74879922013-10-09 14:38:46 +0200820 if self._default is None:
821 self._default = new_default
Jack Neusb8c84482021-06-15 14:28:30 +0000822 elif not emptyDefault and new_default != self._default:
David Pursehouse37128b62013-10-15 10:48:40 +0900823 raise ManifestParseError('duplicate default in %s' %
824 (self.manifestFile))
Julien Campergue74879922013-10-09 14:38:46 +0200825
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700826 if self._default is None:
827 self._default = _Default()
828
Colin Cross23acdd32012-04-21 00:33:54 -0700829 for node in itertools.chain(*node_list):
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700830 if node.nodeName == 'notice':
831 if self._notice is not None:
Doug Anderson37282b42011-03-04 11:54:18 -0800832 raise ManifestParseError(
833 'duplicate notice in %s' %
834 (self.manifestFile))
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700835 self._notice = self._ParseNotice(node)
836
Colin Cross23acdd32012-04-21 00:33:54 -0700837 for node in itertools.chain(*node_list):
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700838 if node.nodeName == 'manifest-server':
839 url = self._reqatt(node, 'url')
840 if self._manifest_server is not None:
David Pursehousec1b86a22012-11-14 11:36:51 +0900841 raise ManifestParseError(
842 'duplicate manifest-server in %s' %
843 (self.manifestFile))
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700844 self._manifest_server = url
845
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800846 def recursively_add_projects(project):
David James8d201162013-10-11 17:03:19 -0700847 projects = self._projects.setdefault(project.name, [])
848 if project.relpath is None:
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800849 raise ManifestParseError(
David James8d201162013-10-11 17:03:19 -0700850 'missing path for %s in %s' %
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800851 (project.name, self.manifestFile))
David James8d201162013-10-11 17:03:19 -0700852 if project.relpath in self._paths:
853 raise ManifestParseError(
854 'duplicate path %s in %s' %
855 (project.relpath, self.manifestFile))
856 self._paths[project.relpath] = project
857 projects.append(project)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800858 for subproject in project.subprojects:
859 recursively_add_projects(subproject)
860
Jack Neusa84f43a2021-09-21 22:23:55 +0000861 repo_hooks_project = None
862 enabled_repo_hooks = None
Colin Cross23acdd32012-04-21 00:33:54 -0700863 for node in itertools.chain(*node_list):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700864 if node.nodeName == 'project':
865 project = self._ParseProject(node)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800866 recursively_add_projects(project)
Josh Triplett884a3872014-06-12 14:57:29 -0700867 if node.nodeName == 'extend-project':
868 name = self._reqatt(node, 'name')
869
870 if name not in self._projects:
871 raise ManifestParseError('extend-project element specifies non-existent '
872 'project: %s' % name)
873
874 path = node.getAttribute('path')
Michael Kelly37c21c22020-06-13 02:10:40 -0700875 dest_path = node.getAttribute('dest-path')
Josh Triplett884a3872014-06-12 14:57:29 -0700876 groups = node.getAttribute('groups')
877 if groups:
Mike Frysinger51e39d52020-12-04 05:32:06 -0500878 groups = self._ParseList(groups)
Luis Hector Chavez7d525852018-03-15 09:54:08 -0700879 revision = node.getAttribute('revision')
Kyunam Jobd0aae92020-02-04 11:38:53 +0900880 remote = node.getAttribute('remote')
881 if remote:
882 remote = self._get_remote(node)
Josh Triplett884a3872014-06-12 14:57:29 -0700883
Michael Kelly37c21c22020-06-13 02:10:40 -0700884 named_projects = self._projects[name]
885 if dest_path and not path and len(named_projects) > 1:
886 raise ManifestParseError('extend-project cannot use dest-path when '
887 'matching multiple projects: %s' % name)
Josh Triplett884a3872014-06-12 14:57:29 -0700888 for p in self._projects[name]:
889 if path and p.relpath != path:
890 continue
891 if groups:
892 p.groups.extend(groups)
Luis Hector Chavez7d525852018-03-15 09:54:08 -0700893 if revision:
Michael Kelly2f3c3312020-07-21 19:40:38 -0700894 p.SetRevision(revision)
895
Kyunam Jobd0aae92020-02-04 11:38:53 +0900896 if remote:
897 p.remote = remote.ToRemoteSpec(name)
Michael Kelly2f3c3312020-07-21 19:40:38 -0700898
Michael Kelly37c21c22020-06-13 02:10:40 -0700899 if dest_path:
900 del self._paths[p.relpath]
901 relpath, worktree, gitdir, objdir, _ = self.GetProjectPaths(name, dest_path)
902 p.UpdatePaths(relpath, worktree, gitdir, objdir)
903 self._paths[p.relpath] = p
904
Doug Anderson37282b42011-03-04 11:54:18 -0800905 if node.nodeName == 'repo-hooks':
Doug Anderson37282b42011-03-04 11:54:18 -0800906 # Only one project can be the hooks project
Jack Neusa84f43a2021-09-21 22:23:55 +0000907 if repo_hooks_project is not None:
Doug Anderson37282b42011-03-04 11:54:18 -0800908 raise ManifestParseError(
909 'duplicate repo-hooks in %s' %
910 (self.manifestFile))
911
Jack Neusa84f43a2021-09-21 22:23:55 +0000912 # Get the name of the project and the (space-separated) list of enabled.
913 repo_hooks_project = self._reqatt(node, 'in-project')
914 enabled_repo_hooks = self._ParseList(self._reqatt(node, 'enabled-list'))
Raman Tenneti1bb4fb22021-01-07 16:50:45 -0800915 if node.nodeName == 'superproject':
916 name = self._reqatt(node, 'name')
917 # There can only be one superproject.
918 if self._superproject.get('name'):
919 raise ManifestParseError(
920 'duplicate superproject in %s' %
921 (self.manifestFile))
922 self._superproject['name'] = name
923 remote_name = node.getAttribute('remote')
924 if not remote_name:
925 remote = self._default.remote
926 else:
927 remote = self._get_remote(node)
928 if remote is None:
929 raise ManifestParseError("no remote for superproject %s within %s" %
930 (name, self.manifestFile))
931 self._superproject['remote'] = remote.ToRemoteSpec(name)
Xin Lie0b16a22021-09-26 23:20:32 -0700932 revision = node.getAttribute('revision') or remote.revision
933 if not revision:
934 revision = self._default.revisionExpr
935 if not revision:
936 raise ManifestParseError('no revision for superproject %s within %s' %
937 (name, self.manifestFile))
938 self._superproject['revision'] = revision
Raman Tenneti1c3f57e2021-05-04 12:32:13 -0700939 if node.nodeName == 'contactinfo':
940 bugurl = self._reqatt(node, 'bugurl')
941 # This element can be repeated, later entries will clobber earlier ones.
Raman Tenneti993af5e2021-05-12 12:00:31 -0700942 self._contactinfo = ContactInfo(bugurl)
943
Colin Cross23acdd32012-04-21 00:33:54 -0700944 if node.nodeName == 'remove-project':
945 name = self._reqatt(node, 'name')
David Jamesb8433df2014-01-30 10:11:17 -0800946
Michael Kelly06da9982021-06-30 01:58:28 -0700947 if name in self._projects:
948 for p in self._projects[name]:
949 del self._paths[p.relpath]
950 del self._projects[name]
951
952 # If the manifest removes the hooks project, treat it as if it deleted
953 # the repo-hooks element too.
Jack Neusa84f43a2021-09-21 22:23:55 +0000954 if repo_hooks_project == name:
955 repo_hooks_project = None
Michael Kelly06da9982021-06-30 01:58:28 -0700956 elif not XmlBool(node, 'optional', False):
David Pursehousef9107482012-11-16 19:12:32 +0900957 raise ManifestParseError('remove-project element specifies non-existent '
958 'project: %s' % name)
Colin Cross23acdd32012-04-21 00:33:54 -0700959
Jack Neusa84f43a2021-09-21 22:23:55 +0000960 # Store repo hooks project information.
961 if repo_hooks_project:
962 # Store a reference to the Project.
963 try:
964 repo_hooks_projects = self._projects[repo_hooks_project]
965 except KeyError:
966 raise ManifestParseError(
967 'project %s not found for repo-hooks' %
968 (repo_hooks_project))
969
970 if len(repo_hooks_projects) != 1:
971 raise ManifestParseError(
972 'internal error parsing repo-hooks in %s' %
973 (self.manifestFile))
974 self._repo_hooks_project = repo_hooks_projects[0]
975 # Store the enabled hooks in the Project object.
976 self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
977
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800978 def _AddMetaProjectMirror(self, m):
979 name = None
980 m_url = m.GetRemote(m.remote.name).url
981 if m_url.endswith('/.git'):
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530982 raise ManifestParseError('refusing to mirror %s' % m_url)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800983
984 if self._default and self._default.remote:
Conley Owensceea3682011-10-20 10:45:47 -0700985 url = self._default.remote.resolvedFetchUrl
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800986 if not url.endswith('/'):
987 url += '/'
988 if m_url.startswith(url):
989 remote = self._default.remote
990 name = m_url[len(url):]
991
992 if name is None:
993 s = m_url.rindex('/') + 1
Conley Owensdb728cd2011-09-26 16:34:01 -0700994 manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
Shawn O. Pearcef35b2d92012-08-02 11:46:22 -0700995 remote = _XmlRemote('origin', fetch=m_url[:s], manifestUrl=manifestUrl)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800996 name = m_url[s:]
997
998 if name.endswith('.git'):
999 name = name[:-4]
1000
1001 if name not in self._projects:
1002 m.PreSync()
1003 gitdir = os.path.join(self.topdir, '%s.git' % name)
David Pursehousee5913ae2020-02-12 13:56:59 +09001004 project = Project(manifest=self,
1005 name=name,
1006 remote=remote.ToRemoteSpec(name),
1007 gitdir=gitdir,
1008 objdir=gitdir,
1009 worktree=None,
1010 relpath=name or None,
1011 revisionExpr=m.revisionExpr,
1012 revisionId=None)
David James8d201162013-10-11 17:03:19 -07001013 self._projects[project.name] = [project]
Kwanhong Leeccd218c2014-02-17 13:07:32 +09001014 self._paths[project.relpath] = project
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001015
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001016 def _ParseRemote(self, node):
1017 """
1018 reads a <remote> element from the manifest file
1019 """
1020 name = self._reqatt(node, 'name')
Yestin Sunb292b982012-07-02 07:32:50 -07001021 alias = node.getAttribute('alias')
1022 if alias == '':
1023 alias = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001024 fetch = self._reqatt(node, 'fetch')
Steve Raed6480452016-08-10 15:00:00 -07001025 pushUrl = node.getAttribute('pushurl')
1026 if pushUrl == '':
1027 pushUrl = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001028 review = node.getAttribute('review')
Shawn O. Pearceae6e0942008-11-06 10:25:35 -08001029 if review == '':
1030 review = None
Anthony King36ea2fb2014-05-06 11:54:01 +01001031 revision = node.getAttribute('revision')
1032 if revision == '':
1033 revision = None
Conley Owensdb728cd2011-09-26 16:34:01 -07001034 manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
Jack Neus6ea0cae2021-07-20 20:52:33 +00001035
1036 remote = _XmlRemote(name, alias, fetch, pushUrl, manifestUrl, review, revision)
1037
1038 for n in node.childNodes:
1039 if n.nodeName == 'annotation':
1040 self._ParseAnnotation(remote, n)
1041
1042 return remote
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001043
1044 def _ParseDefault(self, node):
1045 """
1046 reads a <default> element from the manifest file
1047 """
1048 d = _Default()
1049 d.remote = self._get_remote(node)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001050 d.revisionExpr = node.getAttribute('revision')
1051 if d.revisionExpr == '':
1052 d.revisionExpr = None
Anatol Pomazau79770d22012-04-20 14:41:59 -07001053
Bryan Jacobsf609f912013-05-06 13:36:24 -04001054 d.destBranchExpr = node.getAttribute('dest-branch') or None
Nasser Grainawida403412018-05-04 12:53:29 -06001055 d.upstreamExpr = node.getAttribute('upstream') or None
Bryan Jacobsf609f912013-05-06 13:36:24 -04001056
Mike Frysingerbb8ee7f2020-02-22 05:30:12 -05001057 d.sync_j = XmlInt(node, 'sync-j', 1)
1058 if d.sync_j <= 0:
1059 raise ManifestParseError('%s: sync-j must be greater than 0, not "%s"' %
1060 (self.manifestFile, d.sync_j))
Anatol Pomazau79770d22012-04-20 14:41:59 -07001061
Mike Frysingerbb8ee7f2020-02-22 05:30:12 -05001062 d.sync_c = XmlBool(node, 'sync-c', False)
1063 d.sync_s = XmlBool(node, 'sync-s', False)
1064 d.sync_tags = XmlBool(node, 'sync-tags', True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001065 return d
1066
Doug Anderson2b8db3c2010-11-01 15:08:06 -07001067 def _ParseNotice(self, node):
1068 """
1069 reads a <notice> element from the manifest file
1070
1071 The <notice> element is distinct from other tags in the XML in that the
1072 data is conveyed between the start and end tag (it's not an empty-element
1073 tag).
1074
1075 The white space (carriage returns, indentation) for the notice element is
1076 relevant and is parsed in a way that is based on how python docstrings work.
1077 In fact, the code is remarkably similar to here:
1078 http://www.python.org/dev/peps/pep-0257/
1079 """
1080 # Get the data out of the node...
1081 notice = node.childNodes[0].data
1082
1083 # Figure out minimum indentation, skipping the first line (the same line
1084 # as the <notice> tag)...
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301085 minIndent = sys.maxsize
Doug Anderson2b8db3c2010-11-01 15:08:06 -07001086 lines = notice.splitlines()
1087 for line in lines[1:]:
1088 lstrippedLine = line.lstrip()
1089 if lstrippedLine:
1090 indent = len(line) - len(lstrippedLine)
1091 minIndent = min(indent, minIndent)
1092
1093 # Strip leading / trailing blank lines and also indentation.
1094 cleanLines = [lines[0].strip()]
1095 for line in lines[1:]:
1096 cleanLines.append(line[minIndent:].rstrip())
1097
1098 # Clear completely blank lines from front and back...
1099 while cleanLines and not cleanLines[0]:
1100 del cleanLines[0]
1101 while cleanLines and not cleanLines[-1]:
1102 del cleanLines[-1]
1103
1104 return '\n'.join(cleanLines)
1105
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001106 def _JoinName(self, parent_name, name):
1107 return os.path.join(parent_name, name)
1108
1109 def _UnjoinName(self, parent_name, name):
1110 return os.path.relpath(name, parent_name)
1111
David Pursehousee5913ae2020-02-12 13:56:59 +09001112 def _ParseProject(self, node, parent=None, **extra_proj_attrs):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001113 """
1114 reads a <project> element from the manifest file
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -07001115 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001116 name = self._reqatt(node, 'name')
Mike Frysingera29424e2021-02-25 21:53:49 -05001117 msg = self._CheckLocalPath(name, dir_ok=True)
1118 if msg:
1119 raise ManifestInvalidPathError(
1120 '<project> invalid "name": %s: %s' % (name, msg))
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001121 if parent:
1122 name = self._JoinName(parent.name, name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001123
1124 remote = self._get_remote(node)
1125 if remote is None:
1126 remote = self._default.remote
1127 if remote is None:
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301128 raise ManifestParseError("no remote for project %s within %s" %
David Pursehouseabdf7502020-02-12 14:58:39 +09001129 (name, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001130
Anthony King36ea2fb2014-05-06 11:54:01 +01001131 revisionExpr = node.getAttribute('revision') or remote.revision
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001132 if not revisionExpr:
1133 revisionExpr = self._default.revisionExpr
1134 if not revisionExpr:
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301135 raise ManifestParseError("no revision for project %s within %s" %
David Pursehouseabdf7502020-02-12 14:58:39 +09001136 (name, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001137
1138 path = node.getAttribute('path')
1139 if not path:
1140 path = name
Mike Frysingera29424e2021-02-25 21:53:49 -05001141 else:
Mike Frysinger0458faa2021-03-10 23:35:44 -05001142 # NB: The "." project is handled specially in Project.Sync_LocalHalf.
1143 msg = self._CheckLocalPath(path, dir_ok=True, cwd_dot_ok=True)
Mike Frysingera29424e2021-02-25 21:53:49 -05001144 if msg:
1145 raise ManifestInvalidPathError(
1146 '<project> invalid "path": %s: %s' % (path, msg))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001147
Mike Frysingerbb8ee7f2020-02-22 05:30:12 -05001148 rebase = XmlBool(node, 'rebase', True)
1149 sync_c = XmlBool(node, 'sync-c', False)
1150 sync_s = XmlBool(node, 'sync-s', self._default.sync_s)
1151 sync_tags = XmlBool(node, 'sync-tags', self._default.sync_tags)
Mike Pontillod3153822012-02-28 11:53:24 -08001152
Mike Frysingerbb8ee7f2020-02-22 05:30:12 -05001153 clone_depth = XmlInt(node, 'clone-depth')
1154 if clone_depth is not None and clone_depth <= 0:
1155 raise ManifestParseError('%s: clone-depth must be greater than 0, not "%s"' %
1156 (self.manifestFile, clone_depth))
David Pursehouseede7f122012-11-27 22:25:30 +09001157
Bryan Jacobsf609f912013-05-06 13:36:24 -04001158 dest_branch = node.getAttribute('dest-branch') or self._default.destBranchExpr
1159
Nasser Grainawida403412018-05-04 12:53:29 -06001160 upstream = node.getAttribute('upstream') or self._default.upstreamExpr
Brian Harring14a66742012-09-28 20:21:57 -07001161
Conley Owens971de8e2012-04-16 10:36:08 -07001162 groups = ''
1163 if node.hasAttribute('groups'):
1164 groups = node.getAttribute('groups')
Mike Frysinger51e39d52020-12-04 05:32:06 -05001165 groups = self._ParseList(groups)
Brian Harring7da13142012-06-15 02:24:20 -07001166
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001167 if parent is None:
Mike Frysinger979d5bd2020-02-09 02:28:34 -05001168 relpath, worktree, gitdir, objdir, use_git_worktrees = \
1169 self.GetProjectPaths(name, path)
Shawn O. Pearcecd81dd62012-10-26 12:18:00 -07001170 else:
Mike Frysinger979d5bd2020-02-09 02:28:34 -05001171 use_git_worktrees = False
David James8d201162013-10-11 17:03:19 -07001172 relpath, worktree, gitdir, objdir = \
1173 self.GetSubprojectPaths(parent, name, path)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001174
1175 default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath]
1176 groups.extend(set(default_groups).difference(groups))
Shawn O. Pearcecd81dd62012-10-26 12:18:00 -07001177
Scott Fandb83b1b2013-02-28 09:34:14 +08001178 if self.IsMirror and node.hasAttribute('force-path'):
Mike Frysingerbb8ee7f2020-02-22 05:30:12 -05001179 if XmlBool(node, 'force-path', False):
Scott Fandb83b1b2013-02-28 09:34:14 +08001180 gitdir = os.path.join(self.topdir, '%s.git' % path)
1181
David Pursehousee5913ae2020-02-12 13:56:59 +09001182 project = Project(manifest=self,
1183 name=name,
1184 remote=remote.ToRemoteSpec(name),
1185 gitdir=gitdir,
1186 objdir=objdir,
1187 worktree=worktree,
1188 relpath=relpath,
1189 revisionExpr=revisionExpr,
1190 revisionId=None,
1191 rebase=rebase,
1192 groups=groups,
1193 sync_c=sync_c,
1194 sync_s=sync_s,
1195 sync_tags=sync_tags,
1196 clone_depth=clone_depth,
1197 upstream=upstream,
1198 parent=parent,
1199 dest_branch=dest_branch,
Mike Frysinger979d5bd2020-02-09 02:28:34 -05001200 use_git_worktrees=use_git_worktrees,
Simran Basib9a1b732015-08-20 12:19:28 -07001201 **extra_proj_attrs)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001202
1203 for n in node.childNodes:
Shawn O. Pearce242b5262009-05-19 13:00:29 -07001204 if n.nodeName == 'copyfile':
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001205 self._ParseCopyFile(project, n)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001206 if n.nodeName == 'linkfile':
1207 self._ParseLinkFile(project, n)
James W. Mills24c13082012-04-12 15:04:13 -05001208 if n.nodeName == 'annotation':
1209 self._ParseAnnotation(project, n)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001210 if n.nodeName == 'project':
David Pursehousee5913ae2020-02-12 13:56:59 +09001211 project.subprojects.append(self._ParseProject(n, parent=project))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001212
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001213 return project
1214
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001215 def GetProjectPaths(self, name, path):
Mike Frysingercebf2272020-05-26 01:02:29 -04001216 # The manifest entries might have trailing slashes. Normalize them to avoid
1217 # unexpected filesystem behavior since we do string concatenation below.
1218 path = path.rstrip('/')
1219 name = name.rstrip('/')
Mike Frysinger979d5bd2020-02-09 02:28:34 -05001220 use_git_worktrees = False
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001221 relpath = path
1222 if self.IsMirror:
1223 worktree = None
1224 gitdir = os.path.join(self.topdir, '%s.git' % name)
David James8d201162013-10-11 17:03:19 -07001225 objdir = gitdir
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001226 else:
1227 worktree = os.path.join(self.topdir, path).replace('\\', '/')
1228 gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
Mike Frysinger979d5bd2020-02-09 02:28:34 -05001229 # We allow people to mix git worktrees & non-git worktrees for now.
1230 # This allows for in situ migration of repo clients.
1231 if os.path.exists(gitdir) or not self.UseGitWorktrees:
1232 objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
1233 else:
1234 use_git_worktrees = True
1235 gitdir = os.path.join(self.repodir, 'worktrees', '%s.git' % name)
1236 objdir = gitdir
1237 return relpath, worktree, gitdir, objdir, use_git_worktrees
David James8d201162013-10-11 17:03:19 -07001238
1239 def GetProjectsWithName(self, name):
1240 return self._projects.get(name, [])
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001241
1242 def GetSubprojectName(self, parent, submodule_path):
1243 return os.path.join(parent.name, submodule_path)
1244
1245 def _JoinRelpath(self, parent_relpath, relpath):
1246 return os.path.join(parent_relpath, relpath)
1247
1248 def _UnjoinRelpath(self, parent_relpath, relpath):
1249 return os.path.relpath(relpath, parent_relpath)
1250
David James8d201162013-10-11 17:03:19 -07001251 def GetSubprojectPaths(self, parent, name, path):
Mike Frysingercebf2272020-05-26 01:02:29 -04001252 # The manifest entries might have trailing slashes. Normalize them to avoid
1253 # unexpected filesystem behavior since we do string concatenation below.
1254 path = path.rstrip('/')
1255 name = name.rstrip('/')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001256 relpath = self._JoinRelpath(parent.relpath, path)
1257 gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path)
David James8d201162013-10-11 17:03:19 -07001258 objdir = os.path.join(parent.gitdir, 'subproject-objects', '%s.git' % name)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001259 if self.IsMirror:
1260 worktree = None
1261 else:
1262 worktree = os.path.join(parent.worktree, path).replace('\\', '/')
David James8d201162013-10-11 17:03:19 -07001263 return relpath, worktree, gitdir, objdir
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001264
Mike Frysinger04122b72019-07-31 23:32:58 -04001265 @staticmethod
Mike Frysingera00c5f42021-02-25 18:26:31 -05001266 def _CheckLocalPath(path, dir_ok=False, cwd_dot_ok=False):
1267 """Verify |path| is reasonable for use in filesystem paths.
1268
Mike Frysingera29424e2021-02-25 21:53:49 -05001269 Used with <copyfile> & <linkfile> & <project> elements.
Mike Frysingera00c5f42021-02-25 18:26:31 -05001270
1271 This only validates the |path| in isolation: it does not check against the
1272 current filesystem state. Thus it is suitable as a first-past in a parser.
1273
1274 It enforces a number of constraints:
1275 * No empty paths.
1276 * No "~" in paths.
1277 * No Unicode codepoints that filesystems might elide when normalizing.
1278 * No relative path components like "." or "..".
1279 * No absolute paths.
1280 * No ".git" or ".repo*" path components.
1281
1282 Args:
1283 path: The path name to validate.
1284 dir_ok: Whether |path| may force a directory (e.g. end in a /).
1285 cwd_dot_ok: Whether |path| may be just ".".
1286
1287 Returns:
1288 None if |path| is OK, a failure message otherwise.
1289 """
1290 if not path:
1291 return 'empty paths not allowed'
1292
Mike Frysinger04122b72019-07-31 23:32:58 -04001293 if '~' in path:
1294 return '~ not allowed (due to 8.3 filenames on Windows filesystems)'
1295
Mike Frysingerf69c7ee2021-04-29 23:15:31 -04001296 path_codepoints = set(path)
1297
Mike Frysinger04122b72019-07-31 23:32:58 -04001298 # Some filesystems (like Apple's HFS+) try to normalize Unicode codepoints
1299 # which means there are alternative names for ".git". Reject paths with
1300 # these in it as there shouldn't be any reasonable need for them here.
1301 # The set of codepoints here was cribbed from jgit's implementation:
1302 # https://eclipse.googlesource.com/jgit/jgit/+/9110037e3e9461ff4dac22fee84ef3694ed57648/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java#884
1303 BAD_CODEPOINTS = {
1304 u'\u200C', # ZERO WIDTH NON-JOINER
1305 u'\u200D', # ZERO WIDTH JOINER
1306 u'\u200E', # LEFT-TO-RIGHT MARK
1307 u'\u200F', # RIGHT-TO-LEFT MARK
1308 u'\u202A', # LEFT-TO-RIGHT EMBEDDING
1309 u'\u202B', # RIGHT-TO-LEFT EMBEDDING
1310 u'\u202C', # POP DIRECTIONAL FORMATTING
1311 u'\u202D', # LEFT-TO-RIGHT OVERRIDE
1312 u'\u202E', # RIGHT-TO-LEFT OVERRIDE
1313 u'\u206A', # INHIBIT SYMMETRIC SWAPPING
1314 u'\u206B', # ACTIVATE SYMMETRIC SWAPPING
1315 u'\u206C', # INHIBIT ARABIC FORM SHAPING
1316 u'\u206D', # ACTIVATE ARABIC FORM SHAPING
1317 u'\u206E', # NATIONAL DIGIT SHAPES
1318 u'\u206F', # NOMINAL DIGIT SHAPES
1319 u'\uFEFF', # ZERO WIDTH NO-BREAK SPACE
1320 }
Mike Frysingerf69c7ee2021-04-29 23:15:31 -04001321 if BAD_CODEPOINTS & path_codepoints:
Mike Frysinger04122b72019-07-31 23:32:58 -04001322 # This message is more expansive than reality, but should be fine.
1323 return 'Unicode combining characters not allowed'
1324
Mike Frysingerf69c7ee2021-04-29 23:15:31 -04001325 # Reject newlines as there shouldn't be any legitmate use for them, they'll
1326 # be confusing to users, and they can easily break tools that expect to be
1327 # able to iterate over newline delimited lists. This even applies to our
1328 # own code like .repo/project.list.
1329 if {'\r', '\n'} & path_codepoints:
1330 return 'Newlines not allowed'
1331
Mike Frysinger04122b72019-07-31 23:32:58 -04001332 # Assume paths might be used on case-insensitive filesystems.
1333 path = path.lower()
1334
Mike Frysingerd9254592020-02-19 22:36:26 -05001335 # Split up the path by its components. We can't use os.path.sep exclusively
1336 # as some platforms (like Windows) will convert / to \ and that bypasses all
1337 # our constructed logic here. Especially since manifest authors only use
1338 # / in their paths.
1339 resep = re.compile(r'[/%s]' % re.escape(os.path.sep))
Mike Frysinger0458faa2021-03-10 23:35:44 -05001340 # Strip off trailing slashes as those only produce '' elements, and we use
1341 # parts to look for individual bad components.
1342 parts = resep.split(path.rstrip('/'))
Mike Frysingerd9254592020-02-19 22:36:26 -05001343
Mike Frysingerae625412020-02-10 17:10:03 -05001344 # Some people use src="." to create stable links to projects. Lets allow
1345 # that but reject all other uses of "." to keep things simple.
Mike Frysingera00c5f42021-02-25 18:26:31 -05001346 if not cwd_dot_ok or parts != ['.']:
Mike Frysingerae625412020-02-10 17:10:03 -05001347 for part in set(parts):
1348 if part in {'.', '..', '.git'} or part.startswith('.repo'):
1349 return 'bad component: %s' % (part,)
Mike Frysinger04122b72019-07-31 23:32:58 -04001350
Mike Frysingera00c5f42021-02-25 18:26:31 -05001351 if not dir_ok and resep.match(path[-1]):
Mike Frysinger04122b72019-07-31 23:32:58 -04001352 return 'dirs not allowed'
1353
Mike Frysingerd9254592020-02-19 22:36:26 -05001354 # NB: The two abspath checks here are to handle platforms with multiple
1355 # filesystem path styles (e.g. Windows).
Mike Frysinger04122b72019-07-31 23:32:58 -04001356 norm = os.path.normpath(path)
Mike Frysingerd9254592020-02-19 22:36:26 -05001357 if (norm == '..' or
1358 (len(norm) >= 3 and norm.startswith('..') and resep.match(norm[0])) or
1359 os.path.isabs(norm) or
1360 norm.startswith('/')):
Mike Frysinger04122b72019-07-31 23:32:58 -04001361 return 'path cannot be outside'
1362
1363 @classmethod
1364 def _ValidateFilePaths(cls, element, src, dest):
1365 """Verify |src| & |dest| are reasonable for <copyfile> & <linkfile>.
1366
1367 We verify the path independent of any filesystem state as we won't have a
1368 checkout available to compare to. i.e. This is for parsing validation
1369 purposes only.
1370
1371 We'll do full/live sanity checking before we do the actual filesystem
1372 modifications in _CopyFile/_LinkFile/etc...
1373 """
1374 # |dest| is the file we write to or symlink we create.
1375 # It is relative to the top of the repo client checkout.
1376 msg = cls._CheckLocalPath(dest)
1377 if msg:
1378 raise ManifestInvalidPathError(
1379 '<%s> invalid "dest": %s: %s' % (element, dest, msg))
1380
1381 # |src| is the file we read from or path we point to for symlinks.
1382 # It is relative to the top of the git project checkout.
Mike Frysingera00c5f42021-02-25 18:26:31 -05001383 is_linkfile = element == 'linkfile'
1384 msg = cls._CheckLocalPath(src, dir_ok=is_linkfile, cwd_dot_ok=is_linkfile)
Mike Frysinger04122b72019-07-31 23:32:58 -04001385 if msg:
1386 raise ManifestInvalidPathError(
1387 '<%s> invalid "src": %s: %s' % (element, src, msg))
1388
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001389 def _ParseCopyFile(self, project, node):
1390 src = self._reqatt(node, 'src')
1391 dest = self._reqatt(node, 'dest')
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001392 if not self.IsMirror:
1393 # src is project relative;
Mike Frysinger04122b72019-07-31 23:32:58 -04001394 # dest is relative to the top of the tree.
1395 # We only validate paths if we actually plan to process them.
1396 self._ValidateFilePaths('copyfile', src, dest)
Mike Frysingere6a202f2019-08-02 15:57:57 -04001397 project.AddCopyFile(src, dest, self.topdir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001398
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001399 def _ParseLinkFile(self, project, node):
1400 src = self._reqatt(node, 'src')
1401 dest = self._reqatt(node, 'dest')
1402 if not self.IsMirror:
1403 # src is project relative;
Mike Frysinger04122b72019-07-31 23:32:58 -04001404 # dest is relative to the top of the tree.
1405 # We only validate paths if we actually plan to process them.
1406 self._ValidateFilePaths('linkfile', src, dest)
Mike Frysingere6a202f2019-08-02 15:57:57 -04001407 project.AddLinkFile(src, dest, self.topdir)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001408
Jack Neus6ea0cae2021-07-20 20:52:33 +00001409 def _ParseAnnotation(self, element, node):
James W. Mills24c13082012-04-12 15:04:13 -05001410 name = self._reqatt(node, 'name')
1411 value = self._reqatt(node, 'value')
1412 try:
1413 keep = self._reqatt(node, 'keep').lower()
1414 except ManifestParseError:
1415 keep = "true"
1416 if keep != "true" and keep != "false":
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301417 raise ManifestParseError('optional "keep" attribute must be '
David Pursehouseabdf7502020-02-12 14:58:39 +09001418 '"true" or "false"')
Jack Neus6ea0cae2021-07-20 20:52:33 +00001419 element.AddAnnotation(name, value, keep)
James W. Mills24c13082012-04-12 15:04:13 -05001420
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001421 def _get_remote(self, node):
1422 name = node.getAttribute('remote')
1423 if not name:
1424 return None
1425
1426 v = self._remotes.get(name)
1427 if not v:
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301428 raise ManifestParseError("remote %s not defined in %s" %
David Pursehouseabdf7502020-02-12 14:58:39 +09001429 (name, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001430 return v
1431
1432 def _reqatt(self, node, attname):
1433 """
1434 reads a required attribute from the node.
1435 """
1436 v = node.getAttribute(attname)
1437 if not v:
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301438 raise ManifestParseError("no %s in <%s> within %s" %
David Pursehouseabdf7502020-02-12 14:58:39 +09001439 (attname, node.nodeName, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001440 return v
Julien Camperguedd654222014-01-09 16:21:37 +01001441
1442 def projectsDiff(self, manifest):
1443 """return the projects differences between two manifests.
1444
1445 The diff will be from self to given manifest.
1446
1447 """
1448 fromProjects = self.paths
1449 toProjects = manifest.paths
1450
Anthony King7446c592014-05-06 09:19:39 +01001451 fromKeys = sorted(fromProjects.keys())
1452 toKeys = sorted(toProjects.keys())
Julien Camperguedd654222014-01-09 16:21:37 +01001453
1454 diff = {'added': [], 'removed': [], 'changed': [], 'unreachable': []}
1455
1456 for proj in fromKeys:
David Pursehouseeeff3532020-02-12 11:24:10 +09001457 if proj not in toKeys:
Julien Camperguedd654222014-01-09 16:21:37 +01001458 diff['removed'].append(fromProjects[proj])
1459 else:
1460 fromProj = fromProjects[proj]
1461 toProj = toProjects[proj]
1462 try:
1463 fromRevId = fromProj.GetCommitRevisionId()
1464 toRevId = toProj.GetCommitRevisionId()
1465 except ManifestInvalidRevisionError:
1466 diff['unreachable'].append((fromProj, toProj))
1467 else:
1468 if fromRevId != toRevId:
1469 diff['changed'].append((fromProj, toProj))
1470 toKeys.remove(proj)
1471
1472 for proj in toKeys:
1473 diff['added'].append(toProjects[proj])
1474
1475 return diff
Simran Basib9a1b732015-08-20 12:19:28 -07001476
1477
1478class GitcManifest(XmlManifest):
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -04001479 """Parser for GitC (git-in-the-cloud) manifests."""
Simran Basib9a1b732015-08-20 12:19:28 -07001480
David Pursehousee5913ae2020-02-12 13:56:59 +09001481 def _ParseProject(self, node, parent=None):
Simran Basib9a1b732015-08-20 12:19:28 -07001482 """Override _ParseProject and add support for GITC specific attributes."""
Mike Frysinger5d9c4972021-02-19 13:34:09 -05001483 return super()._ParseProject(
Simran Basib9a1b732015-08-20 12:19:28 -07001484 node, parent=parent, old_revision=node.getAttribute('old-revision'))
1485
1486 def _output_manifest_project_extras(self, p, e):
1487 """Output GITC Specific Project attributes"""
1488 if p.old_revision:
Stefan Beller66851062016-06-17 16:40:08 -07001489 e.setAttribute('old-revision', str(p.old_revision))
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -04001490
1491
1492class RepoClient(XmlManifest):
1493 """Manages a repo client checkout."""
1494
1495 def __init__(self, repodir, manifest_file=None):
1496 self.isGitcClient = False
1497
1498 if os.path.exists(os.path.join(repodir, LOCAL_MANIFEST_NAME)):
1499 print('error: %s is not supported; put local manifests in `%s` instead' %
1500 (LOCAL_MANIFEST_NAME, os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME)),
1501 file=sys.stderr)
1502 sys.exit(1)
1503
1504 if manifest_file is None:
1505 manifest_file = os.path.join(repodir, MANIFEST_FILE_NAME)
1506 local_manifests = os.path.abspath(os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME))
Mike Frysinger5d9c4972021-02-19 13:34:09 -05001507 super().__init__(repodir, manifest_file, local_manifests)
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -04001508
1509 # TODO: Completely separate manifest logic out of the client.
1510 self.manifest = self
1511
1512
1513class GitcClient(RepoClient, GitcManifest):
1514 """Manages a GitC client checkout."""
1515
1516 def __init__(self, repodir, gitc_client_name):
1517 """Initialize the GitcManifest object."""
1518 self.gitc_client_name = gitc_client_name
1519 self.gitc_client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
1520 gitc_client_name)
1521
Mike Frysinger5d9c4972021-02-19 13:34:09 -05001522 super().__init__(repodir, os.path.join(self.gitc_client_dir, '.manifest'))
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -04001523 self.isGitcClient = True