Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 1 | # Copyright (C) 2021 The Android Open Source Project |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 15 | """Provide functionality to get all projects and their commit ids from Superproject. |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 16 | |
| 17 | For more information on superproject, check out: |
| 18 | https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects |
| 19 | |
| 20 | Examples: |
LaMont Jones | ff6b1da | 2022-06-01 21:03:34 +0000 | [diff] [blame] | 21 | superproject = Superproject(manifest, name, remote, revision) |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 22 | UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects) |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 23 | """ |
| 24 | |
Raman Tenneti | ceba2dd | 2021-02-22 16:54:56 -0800 | [diff] [blame] | 25 | import hashlib |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 26 | import functools |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 27 | import os |
| 28 | import sys |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 29 | import time |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 30 | from typing import NamedTuple |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 31 | |
Raman Tenneti | e253b43 | 2021-06-02 10:05:54 -0700 | [diff] [blame] | 32 | from git_command import git_require, GitCommand |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 33 | from git_config import RepoConfig |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 34 | from git_refs import R_HEADS |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 35 | |
Raman Tenneti | 8d43dea | 2021-02-07 16:30:27 -0800 | [diff] [blame] | 36 | _SUPERPROJECT_GIT_NAME = 'superproject.git' |
| 37 | _SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml' |
| 38 | |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 39 | |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 40 | class SyncResult(NamedTuple): |
| 41 | """Return the status of sync and whether caller should exit.""" |
| 42 | |
| 43 | # Whether the superproject sync was successful. |
| 44 | success: bool |
| 45 | # Whether the caller should exit. |
| 46 | fatal: bool |
| 47 | |
| 48 | |
| 49 | class CommitIdsResult(NamedTuple): |
| 50 | """Return the commit ids and whether caller should exit.""" |
| 51 | |
| 52 | # A dictionary with the projects/commit ids on success, otherwise None. |
| 53 | commit_ids: dict |
| 54 | # Whether the caller should exit. |
| 55 | fatal: bool |
| 56 | |
| 57 | |
| 58 | class UpdateProjectsResult(NamedTuple): |
| 59 | """Return the overriding manifest file and whether caller should exit.""" |
| 60 | |
Raman Tenneti | b55769a | 2021-08-13 11:47:24 -0700 | [diff] [blame] | 61 | # Path name of the overriding manifest file if successful, otherwise None. |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 62 | manifest_path: str |
| 63 | # Whether the caller should exit. |
| 64 | fatal: bool |
| 65 | |
| 66 | |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 67 | class Superproject(object): |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 68 | """Get commit ids from superproject. |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 69 | |
Raman Tenneti | ceba2dd | 2021-02-22 16:54:56 -0800 | [diff] [blame] | 70 | Initializes a local copy of a superproject for the manifest. This allows |
| 71 | lookup of commit ids for all projects. It contains _project_commit_ids which |
| 72 | is a dictionary with project/commit id entries. |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 73 | """ |
LaMont Jones | d56e2eb | 2022-04-07 18:14:46 +0000 | [diff] [blame] | 74 | def __init__(self, manifest, name, remote, revision, |
| 75 | superproject_dir='exp-superproject'): |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 76 | """Initializes superproject. |
| 77 | |
| 78 | Args: |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 79 | manifest: A Manifest object that is to be written to a file. |
LaMont Jones | d56e2eb | 2022-04-07 18:14:46 +0000 | [diff] [blame] | 80 | name: The unique name of the superproject |
| 81 | remote: The RemoteSpec for the remote. |
| 82 | revision: The name of the git branch to track. |
| 83 | superproject_dir: Relative path under |manifest.subdir| to checkout |
| 84 | superproject. |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 85 | """ |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 86 | self._project_commit_ids = None |
| 87 | self._manifest = manifest |
LaMont Jones | d56e2eb | 2022-04-07 18:14:46 +0000 | [diff] [blame] | 88 | self.name = name |
| 89 | self.remote = remote |
| 90 | self.revision = self._branch = revision |
| 91 | self._repodir = manifest.repodir |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 92 | self._superproject_dir = superproject_dir |
LaMont Jones | cc879a9 | 2021-11-18 22:40:18 +0000 | [diff] [blame] | 93 | self._superproject_path = manifest.SubmanifestInfoDir(manifest.path_prefix, |
| 94 | superproject_dir) |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 95 | self._manifest_path = os.path.join(self._superproject_path, |
Raman Tenneti | 8d43dea | 2021-02-07 16:30:27 -0800 | [diff] [blame] | 96 | _SUPERPROJECT_MANIFEST_NAME) |
LaMont Jones | d56e2eb | 2022-04-07 18:14:46 +0000 | [diff] [blame] | 97 | git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-' |
| 98 | self._remote_url = remote.url |
Raman Tenneti | ceba2dd | 2021-02-22 16:54:56 -0800 | [diff] [blame] | 99 | self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME |
| 100 | self._work_git = os.path.join(self._superproject_path, self._work_git_name) |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 101 | |
LaMont Jones | ff6b1da | 2022-06-01 21:03:34 +0000 | [diff] [blame] | 102 | # The following are command arguemnts, rather than superproject attributes, |
| 103 | # and were included here originally. They should eventually become |
LaMont Jones | d56e2eb | 2022-04-07 18:14:46 +0000 | [diff] [blame] | 104 | # arguments that are passed down from the public methods, instead of being |
| 105 | # treated as attributes. |
| 106 | self._git_event_log = None |
| 107 | self._quiet = False |
| 108 | self._print_messages = False |
| 109 | |
| 110 | def SetQuiet(self, value): |
| 111 | """Set the _quiet attribute.""" |
| 112 | self._quiet = value |
| 113 | |
| 114 | def SetPrintMessages(self, value): |
| 115 | """Set the _print_messages attribute.""" |
| 116 | self._print_messages = value |
| 117 | |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 118 | @property |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 119 | def project_commit_ids(self): |
| 120 | """Returns a dictionary of projects and their commit ids.""" |
| 121 | return self._project_commit_ids |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 122 | |
Raman Tenneti | ae86a46 | 2021-07-27 08:54:59 -0700 | [diff] [blame] | 123 | @property |
| 124 | def manifest_path(self): |
| 125 | """Returns the manifest path if the path exists or None.""" |
| 126 | return self._manifest_path if os.path.exists(self._manifest_path) else None |
| 127 | |
Raman Tenneti | 5637afc | 2021-08-11 09:26:30 -0700 | [diff] [blame] | 128 | def _LogMessage(self, message): |
Raman Tenneti | 8db30d6 | 2021-07-06 21:30:06 -0700 | [diff] [blame] | 129 | """Logs message to stderr and _git_event_log.""" |
Raman Tenneti | b55769a | 2021-08-13 11:47:24 -0700 | [diff] [blame] | 130 | if self._print_messages: |
| 131 | print(message, file=sys.stderr) |
Raman Tenneti | 7f8bd85 | 2021-09-02 16:13:06 -0700 | [diff] [blame] | 132 | self._git_event_log.ErrorEvent(message, f'{message}') |
Raman Tenneti | 8db30d6 | 2021-07-06 21:30:06 -0700 | [diff] [blame] | 133 | |
Raman Tenneti | d8e8ae8 | 2021-09-15 16:32:33 -0700 | [diff] [blame] | 134 | def _LogMessagePrefix(self): |
| 135 | """Returns the prefix string to be logged in each log message""" |
| 136 | return f'repo superproject branch: {self._branch} url: {self._remote_url}' |
| 137 | |
Raman Tenneti | 5637afc | 2021-08-11 09:26:30 -0700 | [diff] [blame] | 138 | def _LogError(self, message): |
| 139 | """Logs error message to stderr and _git_event_log.""" |
Raman Tenneti | d8e8ae8 | 2021-09-15 16:32:33 -0700 | [diff] [blame] | 140 | self._LogMessage(f'{self._LogMessagePrefix()} error: {message}') |
Raman Tenneti | 5637afc | 2021-08-11 09:26:30 -0700 | [diff] [blame] | 141 | |
| 142 | def _LogWarning(self, message): |
| 143 | """Logs warning message to stderr and _git_event_log.""" |
Raman Tenneti | d8e8ae8 | 2021-09-15 16:32:33 -0700 | [diff] [blame] | 144 | self._LogMessage(f'{self._LogMessagePrefix()} warning: {message}') |
Raman Tenneti | 5637afc | 2021-08-11 09:26:30 -0700 | [diff] [blame] | 145 | |
Raman Tenneti | ceba2dd | 2021-02-22 16:54:56 -0800 | [diff] [blame] | 146 | def _Init(self): |
| 147 | """Sets up a local Git repository to get a copy of a superproject. |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 148 | |
| 149 | Returns: |
Raman Tenneti | ceba2dd | 2021-02-22 16:54:56 -0800 | [diff] [blame] | 150 | True if initialization is successful, or False. |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 151 | """ |
Raman Tenneti | 8d43dea | 2021-02-07 16:30:27 -0800 | [diff] [blame] | 152 | if not os.path.exists(self._superproject_path): |
| 153 | os.mkdir(self._superproject_path) |
Raman Tenneti | ef99ec0 | 2021-03-04 10:29:40 -0800 | [diff] [blame] | 154 | if not self._quiet and not os.path.exists(self._work_git): |
| 155 | print('%s: Performing initial setup for superproject; this might take ' |
| 156 | 'several minutes.' % self._work_git) |
Raman Tenneti | ceba2dd | 2021-02-22 16:54:56 -0800 | [diff] [blame] | 157 | cmd = ['init', '--bare', self._work_git_name] |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 158 | p = GitCommand(None, |
| 159 | cmd, |
| 160 | cwd=self._superproject_path, |
| 161 | capture_stdout=True, |
| 162 | capture_stderr=True) |
| 163 | retval = p.Wait() |
| 164 | if retval: |
Raman Tenneti | 5637afc | 2021-08-11 09:26:30 -0700 | [diff] [blame] | 165 | self._LogWarning(f'git init call failed, command: git {cmd}, ' |
| 166 | f'return code: {retval}, stderr: {p.stderr}') |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 167 | return False |
| 168 | return True |
| 169 | |
Raman Tenneti | d8e8ae8 | 2021-09-15 16:32:33 -0700 | [diff] [blame] | 170 | def _Fetch(self): |
| 171 | """Fetches a local copy of a superproject for the manifest based on |_remote_url|. |
Raman Tenneti | 9e78753 | 2021-02-01 11:47:06 -0800 | [diff] [blame] | 172 | |
| 173 | Returns: |
Raman Tenneti | ceba2dd | 2021-02-22 16:54:56 -0800 | [diff] [blame] | 174 | True if fetch is successful, or False. |
Raman Tenneti | 9e78753 | 2021-02-01 11:47:06 -0800 | [diff] [blame] | 175 | """ |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 176 | if not os.path.exists(self._work_git): |
Raman Tenneti | 5637afc | 2021-08-11 09:26:30 -0700 | [diff] [blame] | 177 | self._LogWarning(f'git fetch missing directory: {self._work_git}') |
Raman Tenneti | 8d43dea | 2021-02-07 16:30:27 -0800 | [diff] [blame] | 178 | return False |
Raman Tenneti | e253b43 | 2021-06-02 10:05:54 -0700 | [diff] [blame] | 179 | if not git_require((2, 28, 0)): |
Raman Tenneti | b55769a | 2021-08-13 11:47:24 -0700 | [diff] [blame] | 180 | self._LogWarning('superproject requires a git version 2.28 or later') |
Raman Tenneti | e253b43 | 2021-06-02 10:05:54 -0700 | [diff] [blame] | 181 | return False |
Raman Tenneti | d8e8ae8 | 2021-09-15 16:32:33 -0700 | [diff] [blame] | 182 | cmd = ['fetch', self._remote_url, '--depth', '1', '--force', '--no-tags', |
| 183 | '--filter', 'blob:none'] |
Raman Tenneti | ceba2dd | 2021-02-22 16:54:56 -0800 | [diff] [blame] | 184 | if self._branch: |
| 185 | cmd += [self._branch + ':' + self._branch] |
Raman Tenneti | 9e78753 | 2021-02-01 11:47:06 -0800 | [diff] [blame] | 186 | p = GitCommand(None, |
| 187 | cmd, |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 188 | cwd=self._work_git, |
Raman Tenneti | 9e78753 | 2021-02-01 11:47:06 -0800 | [diff] [blame] | 189 | capture_stdout=True, |
| 190 | capture_stderr=True) |
| 191 | retval = p.Wait() |
| 192 | if retval: |
Raman Tenneti | 5637afc | 2021-08-11 09:26:30 -0700 | [diff] [blame] | 193 | self._LogWarning(f'git fetch call failed, command: git {cmd}, ' |
| 194 | f'return code: {retval}, stderr: {p.stderr}') |
Raman Tenneti | 9e78753 | 2021-02-01 11:47:06 -0800 | [diff] [blame] | 195 | return False |
| 196 | return True |
| 197 | |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 198 | def _LsTree(self): |
Raman Tenneti | ceba2dd | 2021-02-22 16:54:56 -0800 | [diff] [blame] | 199 | """Gets the commit ids for all projects. |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 200 | |
| 201 | Works only in git repositories. |
| 202 | |
| 203 | Returns: |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 204 | data: data returned from 'git ls-tree ...' instead of None. |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 205 | """ |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 206 | if not os.path.exists(self._work_git): |
Raman Tenneti | 5637afc | 2021-08-11 09:26:30 -0700 | [diff] [blame] | 207 | self._LogWarning(f'git ls-tree missing directory: {self._work_git}') |
Raman Tenneti | 8d43dea | 2021-02-07 16:30:27 -0800 | [diff] [blame] | 208 | return None |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 209 | data = None |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 210 | branch = 'HEAD' if not self._branch else self._branch |
Raman Tenneti | ce64e3d | 2021-02-08 13:27:41 -0800 | [diff] [blame] | 211 | cmd = ['ls-tree', '-z', '-r', branch] |
| 212 | |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 213 | p = GitCommand(None, |
| 214 | cmd, |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 215 | cwd=self._work_git, |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 216 | capture_stdout=True, |
| 217 | capture_stderr=True) |
| 218 | retval = p.Wait() |
| 219 | if retval == 0: |
| 220 | data = p.stdout |
| 221 | else: |
Raman Tenneti | 5637afc | 2021-08-11 09:26:30 -0700 | [diff] [blame] | 222 | self._LogWarning(f'git ls-tree call failed, command: git {cmd}, ' |
| 223 | f'return code: {retval}, stderr: {p.stderr}') |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 224 | return data |
| 225 | |
LaMont Jones | d56e2eb | 2022-04-07 18:14:46 +0000 | [diff] [blame] | 226 | def Sync(self, git_event_log): |
Raman Tenneti | ceba2dd | 2021-02-22 16:54:56 -0800 | [diff] [blame] | 227 | """Gets a local copy of a superproject for the manifest. |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 228 | |
LaMont Jones | d56e2eb | 2022-04-07 18:14:46 +0000 | [diff] [blame] | 229 | Args: |
| 230 | git_event_log: an EventLog, for git tracing. |
| 231 | |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 232 | Returns: |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 233 | SyncResult |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 234 | """ |
LaMont Jones | d56e2eb | 2022-04-07 18:14:46 +0000 | [diff] [blame] | 235 | self._git_event_log = git_event_log |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 236 | if not self._manifest.superproject: |
Raman Tenneti | 5637afc | 2021-08-11 09:26:30 -0700 | [diff] [blame] | 237 | self._LogWarning(f'superproject tag is not defined in manifest: ' |
| 238 | f'{self._manifest.manifestFile}') |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 239 | return SyncResult(False, False) |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 240 | |
LaMont Jones | 2cc3ab7 | 2022-04-13 15:58:58 +0000 | [diff] [blame] | 241 | _PrintBetaNotice() |
| 242 | |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 243 | should_exit = True |
Raman Tenneti | d8e8ae8 | 2021-09-15 16:32:33 -0700 | [diff] [blame] | 244 | if not self._remote_url: |
Raman Tenneti | 5637afc | 2021-08-11 09:26:30 -0700 | [diff] [blame] | 245 | self._LogWarning(f'superproject URL is not defined in manifest: ' |
| 246 | f'{self._manifest.manifestFile}') |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 247 | return SyncResult(False, should_exit) |
Raman Tenneti | 8d43dea | 2021-02-07 16:30:27 -0800 | [diff] [blame] | 248 | |
Raman Tenneti | ceba2dd | 2021-02-22 16:54:56 -0800 | [diff] [blame] | 249 | if not self._Init(): |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 250 | return SyncResult(False, should_exit) |
Raman Tenneti | d8e8ae8 | 2021-09-15 16:32:33 -0700 | [diff] [blame] | 251 | if not self._Fetch(): |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 252 | return SyncResult(False, should_exit) |
Raman Tenneti | ef99ec0 | 2021-03-04 10:29:40 -0800 | [diff] [blame] | 253 | if not self._quiet: |
| 254 | print('%s: Initial setup for superproject completed.' % self._work_git) |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 255 | return SyncResult(True, False) |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 256 | |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 257 | def _GetAllProjectsCommitIds(self): |
| 258 | """Get commit ids for all projects from superproject and save them in _project_commit_ids. |
| 259 | |
| 260 | Returns: |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 261 | CommitIdsResult |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 262 | """ |
LaMont Jones | d56e2eb | 2022-04-07 18:14:46 +0000 | [diff] [blame] | 263 | sync_result = self.Sync(self._git_event_log) |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 264 | if not sync_result.success: |
| 265 | return CommitIdsResult(None, sync_result.fatal) |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 266 | |
| 267 | data = self._LsTree() |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 268 | if not data: |
Raman Tenneti | d8e8ae8 | 2021-09-15 16:32:33 -0700 | [diff] [blame] | 269 | self._LogWarning(f'git ls-tree failed to return data for manifest: ' |
Raman Tenneti | b55769a | 2021-08-13 11:47:24 -0700 | [diff] [blame] | 270 | f'{self._manifest.manifestFile}') |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 271 | return CommitIdsResult(None, True) |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 272 | |
| 273 | # Parse lines like the following to select lines starting with '160000' and |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 274 | # build a dictionary with project path (last element) and its commit id (3rd element). |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 275 | # |
| 276 | # 160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00 |
| 277 | # 120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00 |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 278 | commit_ids = {} |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 279 | for line in data.split('\x00'): |
| 280 | ls_data = line.split(None, 3) |
| 281 | if not ls_data: |
| 282 | break |
| 283 | if ls_data[0] == '160000': |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 284 | commit_ids[ls_data[3]] = ls_data[2] |
Raman Tenneti | 6a872c9 | 2021-01-14 19:17:50 -0800 | [diff] [blame] | 285 | |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 286 | self._project_commit_ids = commit_ids |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 287 | return CommitIdsResult(commit_ids, False) |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 288 | |
Raman Tenneti | b55769a | 2021-08-13 11:47:24 -0700 | [diff] [blame] | 289 | def _WriteManifestFile(self): |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 290 | """Writes manifest to a file. |
| 291 | |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 292 | Returns: |
| 293 | manifest_path: Path name of the file into which manifest is written instead of None. |
| 294 | """ |
| 295 | if not os.path.exists(self._superproject_path): |
Raman Tenneti | 5637afc | 2021-08-11 09:26:30 -0700 | [diff] [blame] | 296 | self._LogWarning(f'missing superproject directory: {self._superproject_path}') |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 297 | return None |
LaMont Jones | a8cf575 | 2022-07-15 20:31:33 +0000 | [diff] [blame] | 298 | manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr(), |
| 299 | omit_local=True).toxml() |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 300 | manifest_path = self._manifest_path |
| 301 | try: |
| 302 | with open(manifest_path, 'w', encoding='utf-8') as fp: |
| 303 | fp.write(manifest_str) |
| 304 | except IOError as e: |
Raman Tenneti | 5637afc | 2021-08-11 09:26:30 -0700 | [diff] [blame] | 305 | self._LogError(f'cannot write manifest to : {manifest_path} {e}') |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 306 | return None |
| 307 | return manifest_path |
| 308 | |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 309 | def _SkipUpdatingProjectRevisionId(self, project): |
| 310 | """Checks if a project's revision id needs to be updated or not. |
| 311 | |
| 312 | Revision id for projects from local manifest will not be updated. |
| 313 | |
| 314 | Args: |
| 315 | project: project whose revision id is being updated. |
| 316 | |
| 317 | Returns: |
| 318 | True if a project's revision id should not be updated, or False, |
| 319 | """ |
| 320 | path = project.relpath |
| 321 | if not path: |
| 322 | return True |
Raman Tenneti | 1da6f30 | 2021-06-28 19:21:38 -0700 | [diff] [blame] | 323 | # Skip the project with revisionId. |
| 324 | if project.revisionId: |
| 325 | return True |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 326 | # Skip the project if it comes from the local manifest. |
LaMont Jones | 87cce68 | 2022-02-14 17:48:31 +0000 | [diff] [blame] | 327 | return project.manifest.IsFromLocalManifest(project) |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 328 | |
LaMont Jones | d56e2eb | 2022-04-07 18:14:46 +0000 | [diff] [blame] | 329 | def UpdateProjectsRevisionId(self, projects, git_event_log): |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 330 | """Update revisionId of every project in projects with the commit id. |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 331 | |
| 332 | Args: |
LaMont Jones | ff6b1da | 2022-06-01 21:03:34 +0000 | [diff] [blame] | 333 | projects: a list of projects whose revisionId needs to be updated. |
| 334 | git_event_log: an EventLog, for git tracing. |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 335 | |
| 336 | Returns: |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 337 | UpdateProjectsResult |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 338 | """ |
LaMont Jones | d56e2eb | 2022-04-07 18:14:46 +0000 | [diff] [blame] | 339 | self._git_event_log = git_event_log |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 340 | commit_ids_result = self._GetAllProjectsCommitIds() |
| 341 | commit_ids = commit_ids_result.commit_ids |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 342 | if not commit_ids: |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 343 | return UpdateProjectsResult(None, commit_ids_result.fatal) |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 344 | |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 345 | projects_missing_commit_ids = [] |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 346 | for project in projects: |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 347 | if self._SkipUpdatingProjectRevisionId(project): |
| 348 | continue |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 349 | path = project.relpath |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 350 | commit_id = commit_ids.get(path) |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 351 | if not commit_id: |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 352 | projects_missing_commit_ids.append(path) |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 353 | |
| 354 | # If superproject doesn't have a commit id for a project, then report an |
| 355 | # error event and continue as if do not use superproject is specified. |
Raman Tenneti | 21dce3d | 2021-02-09 00:26:31 -0800 | [diff] [blame] | 356 | if projects_missing_commit_ids: |
Raman Tenneti | 5637afc | 2021-08-11 09:26:30 -0700 | [diff] [blame] | 357 | self._LogWarning(f'please file a bug using {self._manifest.contactinfo.bugurl} ' |
| 358 | f'to report missing commit_ids for: {projects_missing_commit_ids}') |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 359 | return UpdateProjectsResult(None, False) |
| 360 | |
| 361 | for project in projects: |
| 362 | if not self._SkipUpdatingProjectRevisionId(project): |
| 363 | project.SetRevisionId(commit_ids.get(project.relpath)) |
Raman Tenneti | 1fd7bc2 | 2021-02-04 14:39:38 -0800 | [diff] [blame] | 364 | |
Raman Tenneti | b55769a | 2021-08-13 11:47:24 -0700 | [diff] [blame] | 365 | manifest_path = self._WriteManifestFile() |
Raman Tenneti | 784e16f | 2021-06-11 17:29:45 -0700 | [diff] [blame] | 366 | return UpdateProjectsResult(manifest_path, False) |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 367 | |
| 368 | |
LaMont Jones | 2cc3ab7 | 2022-04-13 15:58:58 +0000 | [diff] [blame] | 369 | @functools.lru_cache(maxsize=10) |
| 370 | def _PrintBetaNotice(): |
| 371 | """Print the notice of beta status.""" |
| 372 | print('NOTICE: --use-superproject is in beta; report any issues to the ' |
| 373 | 'address described in `repo version`', file=sys.stderr) |
| 374 | |
| 375 | |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 376 | @functools.lru_cache(maxsize=None) |
| 377 | def _UseSuperprojectFromConfiguration(): |
| 378 | """Returns the user choice of whether to use superproject.""" |
| 379 | user_cfg = RepoConfig.ForUser() |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 380 | time_now = int(time.time()) |
| 381 | |
| 382 | user_value = user_cfg.GetBoolean('repo.superprojectChoice') |
| 383 | if user_value is not None: |
| 384 | user_expiration = user_cfg.GetInt('repo.superprojectChoiceExpire') |
Xin Li | 0ec2029 | 2021-09-14 16:42:37 -0700 | [diff] [blame] | 385 | if user_expiration is None or user_expiration <= 0 or user_expiration >= time_now: |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 386 | # TODO(b/190688390) - Remove prompt when we are comfortable with the new |
| 387 | # default value. |
Xin Li | 1328c35 | 2021-09-08 00:25:30 -0700 | [diff] [blame] | 388 | if user_value: |
| 389 | print(('You are currently enrolled in Git submodules experiment ' |
| 390 | '(go/android-submodules-quickstart). Use --no-use-superproject ' |
| 391 | 'to override.\n'), file=sys.stderr) |
| 392 | else: |
| 393 | print(('You are not currently enrolled in Git submodules experiment ' |
| 394 | '(go/android-submodules-quickstart). Use --use-superproject ' |
| 395 | 'to override.\n'), file=sys.stderr) |
Xin Li | 6f8c1bf | 2021-09-24 02:15:39 +0000 | [diff] [blame] | 396 | return user_value |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 397 | |
| 398 | # We don't have an unexpired choice, ask for one. |
Raman Tenneti | b55769a | 2021-08-13 11:47:24 -0700 | [diff] [blame] | 399 | system_cfg = RepoConfig.ForSystem() |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 400 | system_value = system_cfg.GetBoolean('repo.superprojectChoice') |
| 401 | if system_value: |
| 402 | # The system configuration is proposing that we should enable the |
Xin Li | 0ec2029 | 2021-09-14 16:42:37 -0700 | [diff] [blame] | 403 | # use of superproject. Treat the user as enrolled for two weeks. |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 404 | # |
| 405 | # TODO(b/190688390) - Remove prompt when we are comfortable with the new |
| 406 | # default value. |
Xin Li | 0ec2029 | 2021-09-14 16:42:37 -0700 | [diff] [blame] | 407 | userchoice = True |
| 408 | time_choiceexpire = time_now + (86400 * 14) |
| 409 | user_cfg.SetString('repo.superprojectChoiceExpire', str(time_choiceexpire)) |
| 410 | user_cfg.SetBoolean('repo.superprojectChoice', userchoice) |
| 411 | print('You are automatically enrolled in Git submodules experiment ' |
| 412 | '(go/android-submodules-quickstart) for another two weeks.\n', |
| 413 | file=sys.stderr) |
| 414 | return True |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 415 | |
| 416 | # For all other cases, we would not use superproject by default. |
| 417 | return False |
| 418 | |
| 419 | |
LaMont Jones | 5fa912b | 2022-04-14 14:41:13 +0000 | [diff] [blame] | 420 | def PrintMessages(use_superproject, manifest): |
| 421 | """Returns a boolean if error/warning messages are to be printed. |
| 422 | |
| 423 | Args: |
| 424 | use_superproject: option value from optparse. |
| 425 | manifest: manifest to use. |
| 426 | """ |
| 427 | return use_superproject is not None or bool(manifest.superproject) |
Raman Tenneti | b55769a | 2021-08-13 11:47:24 -0700 | [diff] [blame] | 428 | |
| 429 | |
LaMont Jones | 5fa912b | 2022-04-14 14:41:13 +0000 | [diff] [blame] | 430 | def UseSuperproject(use_superproject, manifest): |
| 431 | """Returns a boolean if use-superproject option is enabled. |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 432 | |
LaMont Jones | 5fa912b | 2022-04-14 14:41:13 +0000 | [diff] [blame] | 433 | Args: |
| 434 | use_superproject: option value from optparse. |
| 435 | manifest: manifest to use. |
LaMont Jones | ff6b1da | 2022-06-01 21:03:34 +0000 | [diff] [blame] | 436 | |
| 437 | Returns: |
| 438 | Whether the superproject should be used. |
LaMont Jones | 5fa912b | 2022-04-14 14:41:13 +0000 | [diff] [blame] | 439 | """ |
| 440 | |
LaMont Jones | ff6b1da | 2022-06-01 21:03:34 +0000 | [diff] [blame] | 441 | if not manifest.superproject: |
| 442 | # This (sub) manifest does not have a superproject definition. |
| 443 | return False |
| 444 | elif use_superproject is not None: |
LaMont Jones | 5fa912b | 2022-04-14 14:41:13 +0000 | [diff] [blame] | 445 | return use_superproject |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 446 | else: |
LaMont Jones | d82be3e | 2022-04-05 19:30:46 +0000 | [diff] [blame] | 447 | client_value = manifest.manifestProject.use_superproject |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 448 | if client_value is not None: |
| 449 | return client_value |
LaMont Jones | d56e2eb | 2022-04-07 18:14:46 +0000 | [diff] [blame] | 450 | elif manifest.superproject: |
Xin Li | 0cb6e92 | 2021-06-16 10:19:00 -0700 | [diff] [blame] | 451 | return _UseSuperprojectFromConfiguration() |
LaMont Jones | d56e2eb | 2022-04-07 18:14:46 +0000 | [diff] [blame] | 452 | else: |
| 453 | return False |