| #!/usr/bin/python |
| # |
| # Copyright (C) 2021 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| |
| """ |
| Merges upstream files to ojluni. This is done by using git to perform a 3-way |
| merge between the current (base) upstream version, ojluni and the new (target) |
| upstream version. The 3-way merge is needed because ojluni sometimes contains |
| some Android-specific changes from the upstream version. |
| |
| This tool is for libcore maintenance; if you're not maintaining libcore, |
| you won't need it (and might not have access to some of the instructions |
| below). |
| |
| The naming of the repositories (expected, ojluni, 7u40, 8u121-b13, |
| 9b113+, 9+181) is based on the directory name where corresponding |
| snapshots are stored when following the instructions at |
| http://go/libcore-o-verify |
| |
| This script tries to preserve Android changes to upstream code when moving to a |
| newer version. |
| |
| All the work is made in a new directory which is initialized as a git |
| repository. An example of the repository structure, where an update is made |
| from version 9b113+ to 11+28, would be: |
| |
| * 5593705 (HEAD -> main) Merge branch 'ojluni' |
| |\ |
| | * 2effe03 (ojluni) Ojluni commit |
| * | 1bef5f3 Target commit (11+28) |
| |/ |
| * 9ae2fbf Base commit (9b113+) |
| |
| The conflicts during the merge get resolved by git whenever possible. However, |
| sometimes there are conflicts that need to be resolved manually. If that is the |
| case, the script will terminate to allow for the resolving. Once the user has |
| resolved the conflicts, they should rerun the script with the --continue |
| option. |
| |
| Once the merge is complete, the script will copy the merged version back to |
| ojluni within the $ANDROID_BUILD_TOP location. |
| |
| For the script to run correctly, it needs the following environment variables |
| defined: |
| - OJLUNI_UPSTREAMS |
| - ANDROID_BUILD_TOP |
| |
| Possible uses: |
| |
| To merge in changes from a newer version of the upstream using a default |
| working dir created in /tmp: |
| merge-from-upstream -f expected -t 11+28 java/util/concurrent |
| |
| To merge in changes from a newer version of the upstream using a custom |
| working dir: |
| merge-from-upstream -f expected -t 11+28 \ |
| -d $HOME/tmp/ojluni-merge java/util/concurrent |
| |
| To merge in changes for a single file: |
| merge-from-upstream -f 9b113+ -t 11+28 \ |
| java/util/concurrent/atomic/AtomicInteger.java |
| |
| To merge in changes, using a custom folder, that require conflict resolution: |
| merge-from-upstream -f expected -t 11+28 \ |
| -d $HOME/tmp/ojluni-merge \ |
| java/util/concurrent |
| <manually resolve conflicts and add them to git staging> |
| merge-from-upstream --continue \ |
| -d $HOME/tmp/ojluni-merge java/util/concurrent |
| """ |
| |
| import argparse |
| import os |
| import os.path |
| import subprocess |
| import sys |
| import shutil |
| |
| |
| def printerr(msg): |
| sys.stderr.write(msg + "\r\n") |
| |
| |
| def user_check(msg): |
| choice = str(input(msg + " [y/N] ")).strip().lower() |
| if choice[:1] == 'y': |
| return True |
| return False |
| |
| |
| def check_env_vars(): |
| keys = [ |
| 'OJLUNI_UPSTREAMS', |
| 'ANDROID_BUILD_TOP', |
| ] |
| result = True |
| for key in keys: |
| if key not in os.environ: |
| printerr("Unable to run, you must have {} defined".format(key)) |
| result = False |
| return result |
| |
| |
| def get_upstream_path(version, rel_path): |
| upstreams = os.environ['OJLUNI_UPSTREAMS'] |
| return '{}/{}/{}'.format(upstreams, version, rel_path) |
| |
| |
| def get_ojluni_path(rel_path): |
| android_build_top = os.environ['ANDROID_BUILD_TOP'] |
| return '{}/libcore/ojluni/src/main/java/{}'.format( |
| android_build_top, rel_path) |
| |
| |
| def make_copy(src, dst): |
| print("Copy " + src + " -> " + dst) |
| if os.path.isfile(src): |
| if os.path.exists(dst) and os.path.isfile(dst): |
| os.remove(dst) |
| shutil.copy(src, dst) |
| else: |
| shutil.copytree(src, dst, dirs_exist_ok=True) |
| |
| |
| class Repo: |
| def __init__(self, dir): |
| self.dir = dir |
| |
| def init(self): |
| if 0 != subprocess.call(['git', 'init', '-b', 'main', self.dir]): |
| raise RuntimeError( |
| "Unable to initialize working git repository.") |
| subprocess.call(['git', '-C', self.dir, |
| 'config', 'rerere.enabled', 'true']) |
| |
| def commit_all(self, id, msg): |
| if 0 != subprocess.call(['git', '-C', self.dir, 'add', '*']): |
| raise RuntimeError("Unable to add the {} files.".format(id)) |
| if 0 != subprocess.call(['git', '-C', self.dir, 'commit', |
| '-m', msg]): |
| raise RuntimeError("Unable to commit the {} files.".format(id)) |
| |
| def checkout_branch(self, branch, is_new=False): |
| cmd = ['git', '-C', self.dir, 'checkout'] |
| if is_new: |
| cmd.append('-b') |
| cmd.append(branch) |
| if 0 != subprocess.call(cmd): |
| raise RuntimeError("Unable to checkout the {} branch." |
| .format(branch)) |
| |
| def merge(self, branch): |
| """ |
| Tries to merge in a branch and returns True if the merge commit has |
| been created. If there are conflicts to be resolved, this returns |
| False. |
| """ |
| if 0 == subprocess.call(['git', '-C', self.dir, |
| 'merge', branch, '--no-edit']): |
| return True |
| if not self.is_merging(): |
| raise RuntimeError("Unable to run merge for the {} branch." |
| .format(branch)) |
| subprocess.call(['git', '-C', self.dir, 'rerere']) |
| return False |
| |
| def check_resolved_from_cache(self): |
| """ |
| Checks if some conflicts have been resolved by the git rerere tool. The |
| tool only applies the previous resolution, but does not mark the file |
| as resolved afterwards. Therefore this function will go through the |
| unresolved files and see if there are outstanding conflicts. If all |
| conflicts have been resolved, the file gets stages. |
| |
| Returns True if all conflicts are resolved, False otherwise. |
| """ |
| # git diff --check will exit with error if there are conflicts to be |
| # resolved, therefore we need to use check=False option to avoid an |
| # exception to be raised |
| conflict_markers = subprocess.run(['git', '-C', self.dir, |
| 'diff', '--check'], |
| stdout=subprocess.PIPE, |
| check=False).stdout |
| conflicts = subprocess.check_output(['git', '-C', self.dir, 'diff', |
| '--name-only', '--diff-filter=U']) |
| |
| for filename in conflicts.splitlines(): |
| if conflict_markers.find(filename) != -1: |
| print("{} still has conflicts, please resolve manually". |
| format(filename)) |
| else: |
| print("{} has been resolved, staging it".format(filename)) |
| subprocess.call(['git', '-C', self.dir, 'add', filename]) |
| |
| return not self.has_conflicts() |
| |
| def has_changes(self): |
| result = subprocess.check_output(['git', '-C', self.dir, 'status', |
| '--porcelain']) |
| return len(result) != 0 |
| |
| def has_conflicts(self): |
| conflicts = subprocess.check_output(['git', '-C', self.dir, 'diff', |
| '--name-only', '--diff-filter=U']) |
| return len(conflicts) != 0 |
| |
| def is_merging(self): |
| return 0 == subprocess.call(['git', '-C', self.dir, 'rev-parse', |
| '-q', '--verify', 'MERGE_HEAD'], |
| stdout=subprocess.DEVNULL) |
| |
| def complete_merge(self): |
| print("Completing merge in {}".format(self.dir)) |
| subprocess.call(['git', '-C', self.dir, 'rerere']) |
| if 0 != subprocess.call(['git', '-C', self.dir, |
| 'commit', '--no-edit']): |
| raise RuntimeError("Unable to complete the merge in {}." |
| .format(self.dir)) |
| if self.is_merging(): |
| raise RuntimeError( |
| "Merging in {} is not complete".format(self.dir)) |
| |
| def load_resolve_files(self, resolve_dir): |
| print("Loading resolve files from {}".format(resolve_dir)) |
| if not os.path.lexists(resolve_dir): |
| print("Resolve dir {} not found, no resolutions will be used" |
| .format(resolve_dir)) |
| return |
| make_copy(resolve_dir, self.dir + "/.git/rr-cache") |
| |
| def save_resolve_files(self, resolve_dir): |
| print("Saving resolve files to {}".format(resolve_dir)) |
| if not os.path.lexists(resolve_dir): |
| os.makedirs(resolve_dir) |
| make_copy(self.dir + "/.git/rr-cache", resolve_dir) |
| |
| |
| class Merger: |
| def __init__(self, repo_dir, rel_path, resolve_dir): |
| self.repo = Repo(repo_dir) |
| # Have all the source files copied inside a src dir, so we don't have |
| # any issue with copying back the .git dir |
| self.working_dir = repo_dir + "/src" |
| self.rel_path = rel_path |
| self.resolve_dir = resolve_dir |
| |
| def create_working_dir(self): |
| if os.path.lexists(self.repo.dir): |
| if not user_check( |
| '{} already exists. Can it be removed?' |
| .format(self.repo.dir)): |
| raise RuntimeError( |
| 'Will not remove {}. Consider using another ' |
| 'working dir'.format(self.repo.dir)) |
| try: |
| shutil.rmtree(self.repo.dir) |
| except OSError: |
| printerr("Unable to delete {}.".format(self.repo.dir)) |
| raise |
| os.makedirs(self.working_dir) |
| self.repo.init() |
| if self.resolve_dir is not None: |
| self.repo.load_resolve_files(self.resolve_dir) |
| |
| def copy_upstream_files(self, version, msg): |
| full_path = get_upstream_path(version, self.rel_path) |
| make_copy(full_path, self.working_dir) |
| self.repo.commit_all(version, msg) |
| |
| def copy_base_files(self, base_version): |
| self.copy_upstream_files(base_version, |
| 'Base commit ({})'.format(base_version)) |
| |
| def copy_target_files(self, target_version): |
| self.copy_upstream_files(target_version, |
| 'Target commit ({})'.format(target_version)) |
| |
| def copy_ojluni_files(self): |
| full_path = get_ojluni_path(self.rel_path) |
| make_copy(full_path, self.working_dir) |
| if self.repo.has_changes(): |
| self.repo.commit_all('ojluni', 'Ojluni commit') |
| return True |
| else: |
| return False |
| |
| def run_ojluni_merge(self): |
| if self.repo.merge('ojluni'): |
| return |
| if self.repo.check_resolved_from_cache(): |
| self.repo.complete_merge() |
| return |
| raise RuntimeError('\r\nThere are conflicts to be resolved.' |
| '\r\nManually merge the changes and rerun ' |
| 'this script with --continue') |
| |
| def copy_back_to_ojluni(self): |
| # Save any resolutions that were made for future reuse |
| if self.resolve_dir is not None: |
| self.repo.save_resolve_files(self.resolve_dir) |
| |
| src_path = self.working_dir |
| dst_path = get_ojluni_path(self.rel_path) |
| if os.path.isfile(dst_path): |
| src_path += '/' + os.path.basename(self.rel_path) |
| make_copy(src_path, dst_path) |
| |
| def run(self, base_version, target_version): |
| print("Merging {} from {} into ojluni (based on {}). " |
| "Using {} as working dir." |
| .format(self.rel_path, target_version, |
| base_version, self.repo.dir)) |
| self.create_working_dir() |
| self.copy_base_files(base_version) |
| # The ojluni code should be added in its own branch. This is to make |
| # Git perform the 3-way merge once a commit is added with the latest |
| # upstream code. |
| self.repo.checkout_branch('ojluni', is_new=True) |
| merge_needed = self.copy_ojluni_files() |
| self.repo.checkout_branch('main') |
| self.copy_target_files(target_version) |
| if merge_needed: |
| # Runs the merge in the working directory, if some conflicts need |
| # to be resolved manually, then an exception is raised which will |
| # terminate the script, informing the user that manual intervention |
| # is needed. |
| self.run_ojluni_merge() |
| else: |
| print("No merging needed as there were no " |
| "Android-specific changes, forwarding to new version ({})" |
| .format(target_version)) |
| self.copy_back_to_ojluni() |
| |
| def complete_existing_run(self): |
| if self.repo.is_merging(): |
| self.repo.complete_merge() |
| self.copy_back_to_ojluni() |
| |
| |
| def main(): |
| if not check_env_vars(): |
| return |
| |
| upstreams = os.environ['OJLUNI_UPSTREAMS'] |
| repositories = sorted( |
| [d for d in os.listdir(upstreams) |
| if os.path.isdir(os.path.join(upstreams, d))] |
| ) |
| |
| parser = argparse.ArgumentParser( |
| description=''' |
| Merge upstream files from ${OJLUNI_UPSTREAMS} to libcore/ojluni. |
| Needs the base (from) repository as well as the target (to) repository. |
| Repositories can be chosen from: |
| ''' + ' '.join(repositories) + '.', |
| # include default values in help |
| formatter_class=argparse.ArgumentDefaultsHelpFormatter, |
| ) |
| parser.add_argument('-f', '--from', default='expected', |
| choices=repositories, |
| dest='base', |
| help='Repository on which the requested ojluni ' |
| 'files are based.') |
| parser.add_argument('-t', '--to', |
| choices=repositories, |
| dest='target', |
| help='Repository to which the requested ojluni ' |
| 'files will be updated.') |
| parser.add_argument('-d', '--work-dir', default='/tmp/ojluni-merge', |
| help='Path where the merge will be performed. ' |
| 'Any existing files in the path will be removed') |
| parser.add_argument('-r', '--resolve-dir', default=None, |
| dest='resolve_dir', |
| help='Path where the git resolutions are cached. ' |
| 'By default, no cache is used.') |
| parser.add_argument('--continue', action='store_true', dest='proceed', |
| help='Flag to specify after merge conflicts ' |
| 'are resolved') |
| parser.add_argument('rel_path', nargs=1, metavar='<relative_path>', |
| help='File to merge: a relative path below ' |
| 'libcore/ojluni/ which could point to ' |
| 'a file or folder.') |
| args = parser.parse_args() |
| try: |
| merger = Merger(args.work_dir, args.rel_path[0], args.resolve_dir) |
| if args.proceed: |
| merger.complete_existing_run() |
| else: |
| if args.target is None: |
| raise RuntimeError('Please specify the target upstream ' |
| 'version using the -t/--to argument') |
| merger.run(args.base, args.target) |
| except Exception as e: |
| printerr(str(e)) |
| |
| |
| if __name__ == "__main__": |
| main() |