| #!/usr/bin/env python |
| # |
| # Copyright (C) 2019 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| # Sample Usage: |
| # $ python update_profiles.py 500000 git_master ALL --profdata-suffix 2019-04-15 |
| # |
| # Additional/frequently-used arguments: |
| # -b BUG adds a 'Bug: <BUG>' to the commit message when adding the profiles. |
| # --do-not-merge adds a 'DO NOT MERGE' tag to the commit message to restrict |
| # automerge of profiles from release branches. |
| # |
| # Try '-h' for a full list of command line arguments. |
| |
| import argparse |
| import os |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| import zipfile |
| |
| import utils |
| |
| X20_BASE_LOCATION = '/google/data/ro/teams/android-pgo-data' |
| |
| class Benchmark(object): |
| def __init__(self, name): |
| self.name = name |
| |
| def x20_profile_location(self): |
| raise NotImplementedError() |
| |
| def apct_job_name(self): |
| raise NotImplementedError() |
| |
| def profdata_file(self, suffix=''): |
| profdata = os.path.join(self.name, '{}.profdata'.format(self.name)) |
| if suffix: |
| profdata += '.' + suffix |
| return profdata |
| |
| def profraw_files(self): |
| raise NotImplementedError() |
| |
| def merge_profraws(self, profile_dir, output): |
| profraws = [os.path.join(profile_dir, p) for p in self.profraw_files(profile_dir)] |
| utils.run_llvm_profdata(profraws, output) |
| |
| |
| class NativeExeBenchmark(Benchmark): |
| def apct_job_name(self): |
| return 'pgo-collector' |
| |
| def x20_profile_location(self): |
| return os.path.join(X20_BASE_LOCATION, 'raw') |
| |
| def profraw_files(self, profile_dir): |
| if self.name == 'hwui': |
| return ['hwuimacro.profraw', 'hwuimacro_64.profraw', |
| 'hwuimicro.profraw', 'hwuimicro_64.profraw', |
| 'skia_nanobench.profraw', 'skia_nanobench_64.profraw'] |
| elif self.name == 'hwbinder': |
| return ['hwbinder.profraw', 'hwbinder_64.profraw'] |
| |
| |
| class APKBenchmark(Benchmark): |
| def apct_job_name(self): |
| return 'apk-pgo-collector' |
| |
| def x20_profile_location(self): |
| return os.path.join(X20_BASE_LOCATION, 'apk-raw') |
| |
| def profdata_file(self, suffix=''): |
| profdata = os.path.join('art', '{}_arm_arm64.profdata'.format(self.name)) |
| if suffix: |
| profdata += '.' + suffix |
| return profdata |
| |
| def profraw_files(self, profile_dir): |
| return os.listdir(profile_dir) |
| |
| |
| def BenchmarkFactory(benchmark_name): |
| if benchmark_name == 'dex2oat': |
| return APKBenchmark(benchmark_name) |
| elif benchmark_name in ['hwui', 'hwbinder']: |
| return NativeExeBenchmark(benchmark_name) |
| else: |
| raise RuntimeError('Unknown benchmark ' + benchmark_name) |
| |
| |
| def extract_profiles(benchmark, branch, build, output_dir): |
| # The APCT results are stored in |
| # <x20_profile_base>/<branch>/<build>/<apct_job_name>/<arbitrary_invocation_dir>/ |
| # |
| # The PGO files are in _data_local_tmp_<id>.zip in the above directory. |
| |
| profile_base = os.path.join(benchmark.x20_profile_location(), branch, build, |
| benchmark.apct_job_name()) |
| invocation_dirs = os.listdir(profile_base) |
| |
| if len(invocation_dirs) == 0: |
| raise RuntimeError('No invocations found in {}'.format(profile_base)) |
| if len(invocation_dirs) > 1: |
| # TODO Add option to pick/select an invocation from the command line. |
| raise RuntimeError('More than one invocation found in {}'.format(profile_base)) |
| |
| profile_dir = os.path.join(profile_base, invocation_dirs[0]) |
| zipfiles = [f for f in os.listdir(profile_dir) if f.startswith('_data_local_tmp')] |
| |
| if len(zipfiles) != 1: |
| raise RuntimeError('Expected one zipfile in {}. Found {}'.format(profile_dir, |
| len(zipfiles))) |
| |
| zipfile_name = os.path.join(profile_dir, zipfiles[0]) |
| zip_ref = zipfile.ZipFile(zipfile_name) |
| zip_ref.extractall(output_dir) |
| zip_ref.close() |
| |
| |
| KNOWN_BENCHMARKS = ['ALL', 'dex2oat', 'hwui', 'hwbinder'] |
| |
| def parse_args(): |
| """Parses and returns command line arguments.""" |
| parser = argparse.ArgumentParser() |
| |
| parser.add_argument( |
| 'build', metavar='BUILD', |
| help='Build number to pull from the build server.') |
| |
| parser.add_argument( |
| '-b', '--bug', type=int, |
| help='Bug to reference in commit message.') |
| |
| parser.add_argument( |
| '--use-current-branch', action='store_true', |
| help='Do not repo start a new branch for the update.') |
| |
| parser.add_argument( |
| '--add-do-not-merge', action='store_true', |
| help='Add \'DO NOT MERGE\' to the commit message.') |
| |
| parser.add_argument( |
| '--profdata-suffix', type=str, default='', |
| help='Suffix to append to merged profdata file') |
| |
| parser.add_argument( |
| 'branch', metavar='BRANCH', |
| help='Fetch profiles for BRANCH (e.g. git_qt-release)') |
| |
| parser.add_argument( |
| 'benchmark', metavar='BENCHMARK', |
| help='Update profiles for BENCHMARK. Choices are {}'.format(KNOWN_BENCHMARKS)) |
| |
| parser.add_argument( |
| '--skip-cleanup', '-sc', |
| action='store_true', |
| default=False, |
| help='Skip the cleanup, and leave intermediate files (in /tmp/pgo-profiles-*)') |
| |
| return parser.parse_args() |
| |
| |
| def get_current_profile(benchmark): |
| profile = benchmark.profdata_file() |
| dirname, basename = os.path.split(profile) |
| |
| old_profiles = [f for f in os.listdir(dirname) if f.startswith(basename)] |
| if len(old_profiles) == 0: |
| return '' |
| return os.path.join(dirname, old_profiles[0]) |
| |
| |
| def main(): |
| args = parse_args() |
| |
| if args.benchmark == 'ALL': |
| worklist = KNOWN_BENCHMARKS[1:] |
| else: |
| worklist = [args.benchmark] |
| |
| profiles_project = os.path.join(utils.android_build_top(), 'toolchain', |
| 'pgo-profiles') |
| os.chdir(profiles_project) |
| |
| if not args.use_current_branch: |
| branch_name = 'update-profiles-' + args.build |
| utils.check_call(['repo', 'start', branch_name, '.']) |
| |
| for benchmark_name in worklist: |
| benchmark = BenchmarkFactory(benchmark_name) |
| |
| # Existing profile file, which gets 'rm'-ed from 'git' down below. |
| current_profile = get_current_profile(benchmark) |
| |
| # Extract profiles to a temporary directory. After extraction, we |
| # expect to find one subdirectory with profraw files under the temporary |
| # directory. |
| extract_dir = tempfile.mkdtemp(prefix='pgo-profiles-'+benchmark_name) |
| extract_profiles(benchmark, args.branch, args.build, extract_dir) |
| |
| if len(os.listdir(extract_dir)) != 1: |
| raise RuntimeError("Expected one subdir under {}".format(extract_dir)) |
| |
| extract_subdir = os.path.join(extract_dir, os.listdir(extract_dir)[0]) |
| |
| # Merge profiles. |
| profdata = benchmark.profdata_file(args.profdata_suffix) |
| benchmark.merge_profraws(extract_subdir, profdata) |
| |
| # Construct 'git' commit message. |
| message_lines = [ |
| 'Update PGO profiles for {}'.format(benchmark_name), '', |
| 'The profiles are from build {}.'.format(args.build), '' |
| ] |
| |
| if args.add_do_not_merge: |
| message_lines[0] = '[DO NOT MERGE] ' + message_lines[0] |
| |
| if args.bug: |
| message_lines.append('') |
| message_lines.append('Bug: http://b/{}'.format(args.bug)) |
| message_lines.append('Test: Build (TH)') |
| message = '\n'.join(message_lines) |
| |
| # Invoke git: Delete current profile, add new profile and commit these |
| # changes. |
| if current_profile: |
| utils.check_call(['git', 'rm', current_profile]) |
| utils.check_call(['git', 'add', profdata]) |
| utils.check_call(['git', 'commit', '-m', message]) |
| |
| if not args.skip_cleanup: |
| shutil.rmtree(extract_dir) |
| |
| |
| if __name__ == '__main__': |
| main() |