Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 1 | # |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 2 | # Copyright (C) 2018 The Android Open Source Project |
| 3 | # |
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
| 7 | # |
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | # |
| 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
| 15 | """A commandline tool to check and update packages in external/ |
| 16 | |
| 17 | Example usage: |
| 18 | updater.sh checkall |
| 19 | updater.sh update kotlinc |
Chih-Hung Hsieh | dd1915d | 2020-09-29 14:22:12 -0700 | [diff] [blame] | 20 | updater.sh update --refresh --keep_date rust/crates/libc |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 21 | """ |
| 22 | |
| 23 | import argparse |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 24 | import enum |
Joel Galenson | 9ca1a86 | 2021-05-19 10:58:20 -0700 | [diff] [blame] | 25 | import glob |
Haibo Huang | 39aaab6 | 2019-01-25 12:23:03 -0800 | [diff] [blame] | 26 | import json |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 27 | import os |
Haibo Huang | 57a3bdc | 2019-07-09 17:08:14 -0700 | [diff] [blame] | 28 | import sys |
Haibo Huang | 24950e7 | 2018-06-29 14:53:39 -0700 | [diff] [blame] | 29 | import subprocess |
Haibo Huang | 39aaab6 | 2019-01-25 12:23:03 -0800 | [diff] [blame] | 30 | import time |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 31 | from typing import Dict, Iterator, List, Union, Tuple, Type |
| 32 | from pathlib import Path |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 33 | |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 34 | from base_updater import Updater |
Chih-Hung Hsieh | 11cf996 | 2020-03-19 02:02:25 -0700 | [diff] [blame] | 35 | from crates_updater import CratesUpdater |
Haibo Huang | 24950e7 | 2018-06-29 14:53:39 -0700 | [diff] [blame] | 36 | from git_updater import GitUpdater |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 37 | from github_archive_updater import GithubArchiveUpdater |
Haibo Huang | c3c0cd4 | 2019-01-29 15:24:45 -0800 | [diff] [blame] | 38 | import fileutils |
| 39 | import git_utils |
ThiƩbaud Weksteen | 4ac289b | 2020-09-28 15:23:29 +0200 | [diff] [blame] | 40 | # pylint: disable=import-error |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 41 | import metadata_pb2 # type: ignore |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 42 | import updater_utils |
| 43 | |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 44 | UPDATERS: List[Type[Updater]] = [ |
| 45 | CratesUpdater, |
| 46 | GithubArchiveUpdater, |
| 47 | GitUpdater, |
| 48 | ] |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 49 | |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 50 | TMP_BRANCH_NAME = 'tmp_auto_upgrade' |
Haibo Huang | 57a3bdc | 2019-07-09 17:08:14 -0700 | [diff] [blame] | 51 | USE_COLOR = sys.stdout.isatty() |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 52 | |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 53 | |
| 54 | @enum.unique |
| 55 | class Color(enum.Enum): |
| 56 | """Colors for output to console.""" |
| 57 | FRESH = '\x1b[32m' |
| 58 | STALE = '\x1b[31;1m' |
| 59 | ERROR = '\x1b[31m' |
| 60 | |
| 61 | |
| 62 | END_COLOR = '\033[0m' |
| 63 | |
| 64 | |
Haibo Huang | a08fb60 | 2020-05-29 16:24:13 -0700 | [diff] [blame] | 65 | def color_string(string: str, color: Color) -> str: |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 66 | """Changes the color of a string when print to terminal.""" |
Haibo Huang | 57a3bdc | 2019-07-09 17:08:14 -0700 | [diff] [blame] | 67 | if not USE_COLOR: |
| 68 | return string |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 69 | return color.value + string + END_COLOR |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 70 | |
| 71 | |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 72 | def build_updater(proj_path: Path) -> Tuple[Updater, metadata_pb2.MetaData]: |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 73 | """Build updater for a project specified by proj_path. |
| 74 | |
| 75 | Reads and parses METADATA file. And builds updater based on the information. |
| 76 | |
| 77 | Args: |
| 78 | proj_path: Absolute or relative path to the project. |
| 79 | |
| 80 | Returns: |
| 81 | The updater object built. None if there's any error. |
| 82 | """ |
| 83 | |
| 84 | proj_path = fileutils.get_absolute_project_path(proj_path) |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 85 | metadata = fileutils.read_metadata(proj_path) |
| 86 | updater = updater_utils.create_updater(metadata, proj_path, UPDATERS) |
| 87 | return (updater, metadata) |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 88 | |
| 89 | |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 90 | def _do_update(args: argparse.Namespace, updater: Updater, |
| 91 | metadata: metadata_pb2.MetaData) -> None: |
| 92 | full_path = updater.project_path |
Haibo Huang | 39aaab6 | 2019-01-25 12:23:03 -0800 | [diff] [blame] | 93 | |
Haibo Huang | c3c0cd4 | 2019-01-29 15:24:45 -0800 | [diff] [blame] | 94 | if args.branch_and_commit: |
| 95 | git_utils.checkout(full_path, args.remote_name + '/master') |
Haibo Huang | c3c0cd4 | 2019-01-29 15:24:45 -0800 | [diff] [blame] | 96 | git_utils.start_branch(full_path, TMP_BRANCH_NAME) |
| 97 | |
Joel Galenson | 40a5a4a | 2021-05-20 09:50:44 -0700 | [diff] [blame] | 98 | try: |
| 99 | updater.update() |
Haibo Huang | c3c0cd4 | 2019-01-29 15:24:45 -0800 | [diff] [blame] | 100 | |
Joel Galenson | 40a5a4a | 2021-05-20 09:50:44 -0700 | [diff] [blame] | 101 | updated_metadata = metadata_pb2.MetaData() |
| 102 | updated_metadata.CopyFrom(metadata) |
| 103 | updated_metadata.third_party.version = updater.latest_version |
| 104 | for metadata_url in updated_metadata.third_party.url: |
| 105 | if metadata_url == updater.current_url: |
| 106 | metadata_url.CopyFrom(updater.latest_url) |
| 107 | # For Rust crates, replace GIT url with ARCHIVE url |
| 108 | if isinstance(updater, CratesUpdater): |
| 109 | updater.update_metadata(updated_metadata, full_path) |
| 110 | fileutils.write_metadata(full_path, updated_metadata, args.keep_date) |
| 111 | git_utils.add_file(full_path, 'METADATA') |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 112 | |
Joel Galenson | 40a5a4a | 2021-05-20 09:50:44 -0700 | [diff] [blame] | 113 | if args.branch_and_commit: |
| 114 | rel_proj_path = fileutils.get_relative_project_path(full_path) |
| 115 | msg = 'Upgrade {} to {}\n\nTest: make\n'.format( |
| 116 | rel_proj_path, updater.latest_version) |
Joel Galenson | 58f0ddf | 2021-07-14 14:47:40 -0700 | [diff] [blame] | 117 | git_utils.remove_gitmodules(full_path) |
Joel Galenson | 40a5a4a | 2021-05-20 09:50:44 -0700 | [diff] [blame] | 118 | git_utils.add_file(full_path, '*') |
| 119 | git_utils.commit(full_path, msg) |
| 120 | except Exception as err: |
| 121 | if updater.rollback(): |
| 122 | print('Rolled back.') |
| 123 | raise err |
Haibo Huang | c3c0cd4 | 2019-01-29 15:24:45 -0800 | [diff] [blame] | 124 | |
| 125 | if args.push_change: |
Joel Galenson | 58eeae7 | 2021-04-07 12:44:31 -0700 | [diff] [blame] | 126 | git_utils.push(full_path, args.remote_name, updater.has_errors) |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 127 | |
Haibo Huang | 39287b1 | 2019-01-30 15:48:27 -0800 | [diff] [blame] | 128 | if args.branch_and_commit: |
Haibo Huang | 38dda43 | 2019-02-01 15:45:35 -0800 | [diff] [blame] | 129 | git_utils.checkout(full_path, args.remote_name + '/master') |
Haibo Huang | 39287b1 | 2019-01-30 15:48:27 -0800 | [diff] [blame] | 130 | |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 131 | |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 132 | def check_and_update(args: argparse.Namespace, |
| 133 | proj_path: Path, |
| 134 | update_lib=False) -> Union[Updater, str]: |
| 135 | """Checks updates for a project. Prints result on console. |
| 136 | |
| 137 | Args: |
| 138 | args: commandline arguments |
| 139 | proj_path: Absolute or relative path to the project. |
| 140 | update: If false, will only check for new version, but not update. |
| 141 | """ |
| 142 | |
| 143 | try: |
| 144 | rel_proj_path = fileutils.get_relative_project_path(proj_path) |
| 145 | print(f'Checking {rel_proj_path}. ', end='') |
| 146 | updater, metadata = build_updater(proj_path) |
| 147 | updater.check() |
| 148 | |
| 149 | current_ver = updater.current_version |
| 150 | latest_ver = updater.latest_version |
| 151 | print('Current version: {}. Latest version: {}'.format( |
| 152 | current_ver, latest_ver), |
| 153 | end='') |
| 154 | |
| 155 | has_new_version = current_ver != latest_ver |
| 156 | if has_new_version: |
| 157 | print(color_string(' Out of date!', Color.STALE)) |
| 158 | else: |
| 159 | print(color_string(' Up to date.', Color.FRESH)) |
| 160 | |
Chih-Hung Hsieh | dd1915d | 2020-09-29 14:22:12 -0700 | [diff] [blame] | 161 | if update_lib and args.refresh: |
| 162 | print('Refreshing the current version') |
| 163 | updater.use_current_as_latest() |
| 164 | if update_lib and (has_new_version or args.force or args.refresh): |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 165 | _do_update(args, updater, metadata) |
| 166 | return updater |
ThiƩbaud Weksteen | 4ac289b | 2020-09-28 15:23:29 +0200 | [diff] [blame] | 167 | # pylint: disable=broad-except |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 168 | except Exception as err: |
| 169 | print('{} {}.'.format(color_string('Failed.', Color.ERROR), err)) |
| 170 | return str(err) |
| 171 | |
| 172 | |
Joel Galenson | 9ca1a86 | 2021-05-19 10:58:20 -0700 | [diff] [blame] | 173 | def check_and_update_path(args: argparse.Namespace, paths: Iterator[str], |
| 174 | update_lib: bool, |
| 175 | delay: int) -> Dict[str, Dict[str, str]]: |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 176 | results = {} |
| 177 | for path in paths: |
| 178 | res = {} |
Joel Galenson | 9ca1a86 | 2021-05-19 10:58:20 -0700 | [diff] [blame] | 179 | updater = check_and_update(args, Path(path), update_lib) |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 180 | if isinstance(updater, str): |
| 181 | res['error'] = updater |
| 182 | else: |
| 183 | res['current'] = updater.current_version |
| 184 | res['latest'] = updater.latest_version |
| 185 | relative_path = fileutils.get_relative_project_path(Path(path)) |
| 186 | results[str(relative_path)] = res |
| 187 | time.sleep(delay) |
| 188 | return results |
| 189 | |
| 190 | |
| 191 | def _list_all_metadata() -> Iterator[str]: |
| 192 | for path, dirs, files in os.walk(fileutils.EXTERNAL_PATH): |
| 193 | if fileutils.METADATA_FILENAME in files: |
| 194 | # Skip sub directories. |
| 195 | dirs[:] = [] |
| 196 | yield path |
| 197 | dirs.sort(key=lambda d: d.lower()) |
| 198 | |
| 199 | |
Joel Galenson | 0c51025 | 2021-09-02 13:58:51 -0700 | [diff] [blame] | 200 | def get_paths(paths: List[str]) -> List[str]: |
| 201 | """Expand paths via globs.""" |
| 202 | # We want to use glob to get all the paths, so we first convert to absolute. |
| 203 | abs_paths = [fileutils.get_absolute_project_path(Path(path)) |
| 204 | for path in paths] |
Joel Galenson | 3564658 | 2021-09-30 13:32:19 -0700 | [diff] [blame] | 205 | result = [path for abs_path in abs_paths |
| 206 | for path in sorted(glob.glob(str(abs_path)))] |
| 207 | if paths and not result: |
| 208 | print('Could not find any valid paths in %s' % str(paths)) |
| 209 | return result |
Joel Galenson | 0c51025 | 2021-09-02 13:58:51 -0700 | [diff] [blame] | 210 | |
| 211 | |
| 212 | def write_json(json_file: str, results: Dict[str, Dict[str, str]]) -> List[str]: |
| 213 | """Output a JSON report.""" |
| 214 | with Path(json_file).open('w') as res_file: |
| 215 | json.dump(results, res_file, sort_keys=True, indent=4) |
| 216 | |
| 217 | |
Haibo Huang | a08fb60 | 2020-05-29 16:24:13 -0700 | [diff] [blame] | 218 | def check(args: argparse.Namespace) -> None: |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 219 | """Handler for check command.""" |
Joel Galenson | 0c51025 | 2021-09-02 13:58:51 -0700 | [diff] [blame] | 220 | paths = _list_all_metadata() if args.all else get_paths(args.paths) |
Joel Galenson | 9ca1a86 | 2021-05-19 10:58:20 -0700 | [diff] [blame] | 221 | results = check_and_update_path(args, paths, False, args.delay) |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 222 | |
| 223 | if args.json_output is not None: |
Joel Galenson | 0c51025 | 2021-09-02 13:58:51 -0700 | [diff] [blame] | 224 | write_json(args.json_output, results) |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 225 | |
| 226 | |
| 227 | def update(args: argparse.Namespace) -> None: |
| 228 | """Handler for update command.""" |
Joel Galenson | 0c51025 | 2021-09-02 13:58:51 -0700 | [diff] [blame] | 229 | all_paths = get_paths(args.paths) |
Joel Galenson | 473af4f | 2021-05-19 11:15:37 -0700 | [diff] [blame] | 230 | # Remove excluded paths. |
| 231 | excludes = set() if args.exclude is None else set(args.exclude) |
| 232 | filtered_paths = [path for path in all_paths |
| 233 | if not Path(path).name in excludes] |
Joel Galenson | 9ca1a86 | 2021-05-19 10:58:20 -0700 | [diff] [blame] | 234 | # Now we can update each path. |
Joel Galenson | 473af4f | 2021-05-19 11:15:37 -0700 | [diff] [blame] | 235 | results = check_and_update_path(args, filtered_paths, True, 0) |
Joel Galenson | 9ca1a86 | 2021-05-19 10:58:20 -0700 | [diff] [blame] | 236 | |
| 237 | if args.json_output is not None: |
Joel Galenson | 0c51025 | 2021-09-02 13:58:51 -0700 | [diff] [blame] | 238 | write_json(args.json_output, results) |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 239 | |
| 240 | |
| 241 | def parse_args() -> argparse.Namespace: |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 242 | """Parses commandline arguments.""" |
| 243 | |
| 244 | parser = argparse.ArgumentParser( |
| 245 | description='Check updates for third party projects in external/.') |
| 246 | subparsers = parser.add_subparsers(dest='cmd') |
| 247 | subparsers.required = True |
| 248 | |
| 249 | # Creates parser for check command. |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 250 | check_parser = subparsers.add_parser('check', |
| 251 | help='Check update for one project.') |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 252 | check_parser.add_argument( |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 253 | 'paths', |
| 254 | nargs='*', |
Haibo Huang | 39aaab6 | 2019-01-25 12:23:03 -0800 | [diff] [blame] | 255 | help='Paths of the project. ' |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 256 | 'Relative paths will be resolved from external/.') |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 257 | check_parser.add_argument('--json_output', |
| 258 | help='Path of a json file to write result to.') |
Haibo Huang | 39aaab6 | 2019-01-25 12:23:03 -0800 | [diff] [blame] | 259 | check_parser.add_argument( |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 260 | '--all', |
| 261 | action='store_true', |
Haibo Huang | 39aaab6 | 2019-01-25 12:23:03 -0800 | [diff] [blame] | 262 | help='If set, check updates for all supported projects.') |
| 263 | check_parser.add_argument( |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 264 | '--delay', |
| 265 | default=0, |
| 266 | type=int, |
Haibo Huang | 39aaab6 | 2019-01-25 12:23:03 -0800 | [diff] [blame] | 267 | help='Time in seconds to wait between checking two projects.') |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 268 | check_parser.set_defaults(func=check) |
| 269 | |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 270 | # Creates parser for update command. |
| 271 | update_parser = subparsers.add_parser('update', help='Update one project.') |
| 272 | update_parser.add_argument( |
Joel Galenson | 9ca1a86 | 2021-05-19 10:58:20 -0700 | [diff] [blame] | 273 | 'paths', |
| 274 | nargs='*', |
| 275 | help='Paths of the project as globs. ' |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 276 | 'Relative paths will be resolved from external/.') |
Joel Galenson | 9ca1a86 | 2021-05-19 10:58:20 -0700 | [diff] [blame] | 277 | update_parser.add_argument('--json_output', |
| 278 | help='Path of a json file to write result to.') |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 279 | update_parser.add_argument( |
| 280 | '--force', |
| 281 | help='Run update even if there\'s no new version.', |
| 282 | action='store_true') |
Chih-Hung Hsieh | dd1915d | 2020-09-29 14:22:12 -0700 | [diff] [blame] | 283 | update_parser.add_argument( |
| 284 | '--refresh', |
| 285 | help='Run update and refresh to the current version.', |
| 286 | action='store_true') |
| 287 | update_parser.add_argument( |
| 288 | '--keep_date', |
| 289 | help='Run update and do not change date in METADATA.', |
| 290 | action='store_true') |
Haibo Huang | 329e681 | 2020-05-29 14:12:20 -0700 | [diff] [blame] | 291 | update_parser.add_argument('--branch_and_commit', |
| 292 | action='store_true', |
| 293 | help='Starts a new branch and commit changes.') |
| 294 | update_parser.add_argument('--push_change', |
| 295 | action='store_true', |
| 296 | help='Pushes change to Gerrit.') |
| 297 | update_parser.add_argument('--remote_name', |
| 298 | default='aosp', |
| 299 | required=False, |
| 300 | help='Upstream remote name.') |
Joel Galenson | 473af4f | 2021-05-19 11:15:37 -0700 | [diff] [blame] | 301 | update_parser.add_argument('--exclude', |
| 302 | action='append', |
| 303 | help='Names of projects to exclude. ' |
| 304 | 'These are just the final part of the path ' |
| 305 | 'with no directories.') |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 306 | update_parser.set_defaults(func=update) |
| 307 | |
| 308 | return parser.parse_args() |
| 309 | |
| 310 | |
Haibo Huang | a08fb60 | 2020-05-29 16:24:13 -0700 | [diff] [blame] | 311 | def main() -> None: |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 312 | """The main entry.""" |
| 313 | |
| 314 | args = parse_args() |
| 315 | args.func(args) |
| 316 | |
| 317 | |
| 318 | if __name__ == '__main__': |
| 319 | main() |