Mike Frysinger | f601376 | 2019-06-13 02:30:51 -0400 | [diff] [blame] | 1 | # -*- coding:utf-8 -*- |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 2 | # |
| 3 | # Copyright (C) 2015 The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
| 17 | from __future__ import print_function |
| 18 | import os |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 19 | import platform |
| 20 | import re |
David Pursehouse | 46496d8 | 2015-08-20 16:37:09 +0900 | [diff] [blame] | 21 | import sys |
Simran Basi | b9a1b73 | 2015-08-20 12:19:28 -0700 | [diff] [blame] | 22 | import time |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 23 | |
| 24 | import git_command |
| 25 | import git_config |
Simran Basi | 8ce5041 | 2015-08-28 14:25:44 -0700 | [diff] [blame] | 26 | import wrapper |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 27 | |
Xin Li | f97e72e | 2016-06-24 12:17:14 -0700 | [diff] [blame] | 28 | from error import ManifestParseError |
| 29 | |
Dan Willemsen | 39252ba | 2016-08-23 14:06:59 -0700 | [diff] [blame] | 30 | NUM_BATCH_RETRIEVE_REVISIONID = 32 |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 31 | |
David Pursehouse | 819827a | 2020-02-12 15:20:19 +0900 | [diff] [blame^] | 32 | |
Simran Basi | 8ce5041 | 2015-08-28 14:25:44 -0700 | [diff] [blame] | 33 | def get_gitc_manifest_dir(): |
| 34 | return wrapper.Wrapper().get_gitc_manifest_dir() |
| 35 | |
David Pursehouse | 819827a | 2020-02-12 15:20:19 +0900 | [diff] [blame^] | 36 | |
Simran Basi | b9a1b73 | 2015-08-20 12:19:28 -0700 | [diff] [blame] | 37 | def parse_clientdir(gitc_fs_path): |
Dan Willemsen | 745b4ad | 2015-10-06 15:23:19 -0700 | [diff] [blame] | 38 | return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path) |
Simran Basi | b9a1b73 | 2015-08-20 12:19:28 -0700 | [diff] [blame] | 39 | |
David Pursehouse | 819827a | 2020-02-12 15:20:19 +0900 | [diff] [blame^] | 40 | |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 41 | def _set_project_revisions(projects): |
| 42 | """Sets the revisionExpr for a list of projects. |
| 43 | |
| 44 | Because of the limit of open file descriptors allowed, length of projects |
| 45 | should not be overly large. Recommend calling this function multiple times |
| 46 | with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects. |
| 47 | |
| 48 | @param projects: List of project objects to set the revionExpr for. |
| 49 | """ |
| 50 | # Retrieve the commit id for each project based off of it's current |
| 51 | # revisionExpr and it is not already a commit id. |
| 52 | project_gitcmds = [( |
| 53 | project, git_command.GitCommand(None, |
| 54 | ['ls-remote', |
| 55 | project.remote.url, |
| 56 | project.revisionExpr], |
| 57 | capture_stdout=True, cwd='/tmp')) |
David Pursehouse | abdf750 | 2020-02-12 14:58:39 +0900 | [diff] [blame] | 58 | for project in projects if not git_config.IsId(project.revisionExpr)] |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 59 | for proj, gitcmd in project_gitcmds: |
| 60 | if gitcmd.Wait(): |
David Pursehouse | 022a1d4 | 2015-08-20 16:41:04 +0900 | [diff] [blame] | 61 | print('FATAL: Failed to retrieve revisionExpr for %s' % proj) |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 62 | sys.exit(1) |
Xin Li | f97e72e | 2016-06-24 12:17:14 -0700 | [diff] [blame] | 63 | revisionExpr = gitcmd.stdout.split('\t')[0] |
| 64 | if not revisionExpr: |
Mike Frysinger | 31067c0 | 2019-06-13 02:13:23 -0400 | [diff] [blame] | 65 | raise ManifestParseError('Invalid SHA-1 revision project %s (%s)' % |
| 66 | (proj.remote.url, proj.revisionExpr)) |
Xin Li | f97e72e | 2016-06-24 12:17:14 -0700 | [diff] [blame] | 67 | proj.revisionExpr = revisionExpr |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 68 | |
David Pursehouse | 819827a | 2020-02-12 15:20:19 +0900 | [diff] [blame^] | 69 | |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 70 | def _manifest_groups(manifest): |
| 71 | """Returns the manifest group string that should be synced |
| 72 | |
| 73 | This is the same logic used by Command.GetProjects(), which is used during |
| 74 | repo sync |
| 75 | |
| 76 | @param manifest: The XmlManifest object |
| 77 | """ |
| 78 | mp = manifest.manifestProject |
| 79 | groups = mp.config.GetString('manifest.groups') |
| 80 | if not groups: |
| 81 | groups = 'default,platform-' + platform.system().lower() |
| 82 | return groups |
| 83 | |
David Pursehouse | 819827a | 2020-02-12 15:20:19 +0900 | [diff] [blame^] | 84 | |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 85 | def generate_gitc_manifest(gitc_manifest, manifest, paths=None): |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 86 | """Generate a manifest for shafsd to use for this GITC client. |
| 87 | |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 88 | @param gitc_manifest: Current gitc manifest, or None if there isn't one yet. |
| 89 | @param manifest: A GitcManifest object loaded with the current repo manifest. |
| 90 | @param paths: List of project paths we want to update. |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 91 | """ |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 92 | |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 93 | print('Generating GITC Manifest by fetching revision SHAs for each ' |
| 94 | 'project.') |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 95 | if paths is None: |
Mike Frysinger | 31067c0 | 2019-06-13 02:13:23 -0400 | [diff] [blame] | 96 | paths = list(manifest.paths.keys()) |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 97 | |
| 98 | groups = [x for x in re.split(r'[,\s]+', _manifest_groups(manifest)) if x] |
| 99 | |
| 100 | # Convert the paths to projects, and filter them to the matched groups. |
| 101 | projects = [manifest.paths[p] for p in paths] |
| 102 | projects = [p for p in projects if p.MatchesGroups(groups)] |
| 103 | |
| 104 | if gitc_manifest is not None: |
Mike Frysinger | 31067c0 | 2019-06-13 02:13:23 -0400 | [diff] [blame] | 105 | for path, proj in manifest.paths.items(): |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 106 | if not proj.MatchesGroups(groups): |
| 107 | continue |
| 108 | |
| 109 | if not proj.upstream and not git_config.IsId(proj.revisionExpr): |
| 110 | proj.upstream = proj.revisionExpr |
| 111 | |
David Pursehouse | eeff353 | 2020-02-12 11:24:10 +0900 | [diff] [blame] | 112 | if path not in gitc_manifest.paths: |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 113 | # Any new projects need their first revision, even if we weren't asked |
| 114 | # for them. |
| 115 | projects.append(proj) |
David Pursehouse | eeff353 | 2020-02-12 11:24:10 +0900 | [diff] [blame] | 116 | elif path not in paths: |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 117 | # And copy revisions from the previous manifest if we're not updating |
| 118 | # them now. |
| 119 | gitc_proj = gitc_manifest.paths[path] |
| 120 | if gitc_proj.old_revision: |
| 121 | proj.revisionExpr = None |
| 122 | proj.old_revision = gitc_proj.old_revision |
| 123 | else: |
| 124 | proj.revisionExpr = gitc_proj.revisionExpr |
| 125 | |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 126 | index = 0 |
Simran Basi | b9a1b73 | 2015-08-20 12:19:28 -0700 | [diff] [blame] | 127 | while index < len(projects): |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 128 | _set_project_revisions( |
David Pursehouse | 54a4e60 | 2020-02-12 14:31:05 +0900 | [diff] [blame] | 129 | projects[index:(index + NUM_BATCH_RETRIEVE_REVISIONID)]) |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 130 | index += NUM_BATCH_RETRIEVE_REVISIONID |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 131 | |
| 132 | if gitc_manifest is not None: |
Mike Frysinger | 31067c0 | 2019-06-13 02:13:23 -0400 | [diff] [blame] | 133 | for path, proj in gitc_manifest.paths.items(): |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 134 | if proj.old_revision and path in paths: |
| 135 | # If we updated a project that has been started, keep the old-revision |
| 136 | # updated. |
| 137 | repo_proj = manifest.paths[path] |
| 138 | repo_proj.old_revision = repo_proj.revisionExpr |
| 139 | repo_proj.revisionExpr = None |
| 140 | |
| 141 | # Convert URLs from relative to absolute. |
Mike Frysinger | 31067c0 | 2019-06-13 02:13:23 -0400 | [diff] [blame] | 142 | for _name, remote in manifest.remotes.items(): |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 143 | remote.fetchUrl = remote.resolvedFetchUrl |
| 144 | |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 145 | # Save the manifest. |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 146 | save_manifest(manifest) |
Simran Basi | b9a1b73 | 2015-08-20 12:19:28 -0700 | [diff] [blame] | 147 | |
David Pursehouse | 819827a | 2020-02-12 15:20:19 +0900 | [diff] [blame^] | 148 | |
Simran Basi | b9a1b73 | 2015-08-20 12:19:28 -0700 | [diff] [blame] | 149 | def save_manifest(manifest, client_dir=None): |
| 150 | """Save the manifest file in the client_dir. |
| 151 | |
| 152 | @param client_dir: Client directory to save the manifest in. |
| 153 | @param manifest: Manifest object to save. |
| 154 | """ |
| 155 | if not client_dir: |
| 156 | client_dir = manifest.gitc_client_dir |
Simran Basi | bdb5271 | 2015-08-10 13:23:23 -0700 | [diff] [blame] | 157 | with open(os.path.join(client_dir, '.manifest'), 'w') as f: |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 158 | manifest.Save(f, groups=_manifest_groups(manifest)) |
Simran Basi | b9a1b73 | 2015-08-20 12:19:28 -0700 | [diff] [blame] | 159 | # TODO(sbasi/jorg): Come up with a solution to remove the sleep below. |
| 160 | # Give the GITC filesystem time to register the manifest changes. |
Dan Willemsen | 5ea32d1 | 2015-09-08 13:27:20 -0700 | [diff] [blame] | 161 | time.sleep(3) |