| #!/usr/bin/env python3 |
| # |
| # Copyright 2018, The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| # |
| # |
| # Measure application start-up time by launching applications under various combinations. |
| # See --help for more details. |
| # |
| # |
| # Sample usage: |
| # $> ./app_startup_runner.py -p com.google.android.calculator -r warm -r cold -lc 10 -o out.csv |
| # $> ./analyze_metrics.py out.csv |
| # |
| # |
| |
| import argparse |
| import csv |
| import itertools |
| import os |
| import sys |
| import tempfile |
| from datetime import timedelta |
| from typing import Any, Callable, Iterable, List, NamedTuple, TextIO, Tuple, \ |
| TypeVar, Union, Optional |
| |
| # local import |
| DIR = os.path.abspath(os.path.dirname(__file__)) |
| sys.path.append(os.path.dirname(DIR)) |
| import lib.cmd_utils as cmd_utils |
| import lib.print_utils as print_utils |
| from app_startup.run_app_with_prefetch import PrefetchAppRunner |
| import app_startup.lib.args_utils as args_utils |
| from app_startup.lib.data_frame import DataFrame |
| from app_startup.lib.perfetto_trace_collector import PerfettoTraceCollector |
| from iorap.compiler import CompilerType |
| import iorap.compiler as compiler |
| |
| # The following command line options participate in the combinatorial generation. |
| # All other arguments have a global effect. |
| _COMBINATORIAL_OPTIONS = ['package', 'readahead', 'compiler_filter', |
| 'activity', 'trace_duration'] |
| _TRACING_READAHEADS = ['mlock', 'fadvise'] |
| _FORWARD_OPTIONS = {'loop_count': '--count'} |
| _RUN_SCRIPT = os.path.join(os.path.dirname(os.path.realpath(__file__)), |
| 'run_app_with_prefetch.py') |
| |
| CollectorPackageInfo = NamedTuple('CollectorPackageInfo', |
| [('package', str), ('compiler_filter', str)]) |
| # by 2; systrace starts up slowly. |
| |
| _UNLOCK_SCREEN_SCRIPT = os.path.join( |
| os.path.dirname(os.path.realpath(__file__)), 'unlock_screen') |
| |
| RunCommandArgs = NamedTuple('RunCommandArgs', |
| [('package', str), |
| ('readahead', str), |
| ('activity', Optional[str]), |
| ('compiler_filter', Optional[str]), |
| ('timeout', Optional[int]), |
| ('debug', bool), |
| ('simulate', bool), |
| ('input', Optional[str]), |
| ('trace_duration', Optional[timedelta])]) |
| |
| # This must be the only mutable global variable. All other global variables are constants to avoid magic literals. |
| _debug = False # See -d/--debug flag. |
| _DEBUG_FORCE = None # Ignore -d/--debug if this is not none. |
| _PERFETTO_TRACE_DURATION_MS = 5000 # milliseconds |
| _PERFETTO_TRACE_DURATION = timedelta(milliseconds=_PERFETTO_TRACE_DURATION_MS) |
| |
| # Type hinting names. |
| T = TypeVar('T') |
| NamedTupleMeta = Callable[ |
| ..., T] # approximation of a (S : NamedTuple<T> where S() == T) metatype. |
| |
| def parse_options(argv: List[str] = None): |
| """Parse command line arguments and return an argparse Namespace object.""" |
| parser = argparse.ArgumentParser(description="Run one or more Android " |
| "applications under various " |
| "settings in order to measure " |
| "startup time.") |
| # argparse considers args starting with - and -- optional in --help, even though required=True. |
| # by using a named argument group --help will clearly say that it's required instead of optional. |
| required_named = parser.add_argument_group('required named arguments') |
| required_named.add_argument('-p', '--package', action='append', |
| dest='packages', |
| help='package of the application', required=True) |
| required_named.add_argument('-r', '--readahead', action='append', |
| dest='readaheads', |
| help='which readahead mode to use', |
| choices=('warm', 'cold', 'mlock', 'fadvise'), |
| required=True) |
| |
| # optional arguments |
| # use a group here to get the required arguments to appear 'above' the optional arguments in help. |
| optional_named = parser.add_argument_group('optional named arguments') |
| optional_named.add_argument('-c', '--compiler-filter', action='append', |
| dest='compiler_filters', |
| help='which compiler filter to use. if omitted it does not enforce the app\'s compiler filter', |
| choices=('speed', 'speed-profile', 'quicken')) |
| optional_named.add_argument('-s', '--simulate', dest='simulate', |
| action='store_true', |
| help='Print which commands will run, but don\'t run the apps') |
| optional_named.add_argument('-d', '--debug', dest='debug', |
| action='store_true', |
| help='Add extra debugging output') |
| optional_named.add_argument('-o', '--output', dest='output', action='store', |
| help='Write CSV output to file.') |
| optional_named.add_argument('-t', '--timeout', dest='timeout', action='store', |
| type=int, default=10, |
| help='Timeout after this many seconds when executing a single run.') |
| optional_named.add_argument('-lc', '--loop-count', dest='loop_count', |
| default=1, type=int, action='store', |
| help='How many times to loop a single run.') |
| optional_named.add_argument('-in', '--inodes', dest='inodes', type=str, |
| action='store', |
| help='Path to inodes file (system/extras/pagecache/pagecache.py -d inodes)') |
| optional_named.add_argument('--compiler-trace-duration-ms', |
| dest='trace_duration', |
| type=lambda ms_str: timedelta(milliseconds=int(ms_str)), |
| action='append', |
| help='The trace duration (milliseconds) in ' |
| 'compilation') |
| optional_named.add_argument('--compiler-type', dest='compiler_type', |
| type=CompilerType, choices=list(CompilerType), |
| default=CompilerType.DEVICE, |
| help='The type of compiler.') |
| |
| return parser.parse_args(argv) |
| |
| def key_to_cmdline_flag(key: str) -> str: |
| """Convert key into a command line flag, e.g. 'foo-bars' -> '--foo-bar' """ |
| if key.endswith("s"): |
| key = key[:-1] |
| return "--" + key.replace("_", "-") |
| |
| def as_run_command(tpl: NamedTuple) -> List[Union[str, Any]]: |
| """ |
| Convert a named tuple into a command-line compatible arguments list. |
| |
| Example: ABC(1, 2, 3) -> ['--a', 1, '--b', 2, '--c', 3] |
| """ |
| args = [] |
| for key, value in tpl._asdict().items(): |
| if value is None: |
| continue |
| args.append(key_to_cmdline_flag(key)) |
| args.append(value) |
| return args |
| |
| def run_perfetto_collector(collector_info: CollectorPackageInfo, |
| timeout: int, |
| simulate: bool) -> Tuple[bool, TextIO]: |
| """Run collector to collect prefetching trace. |
| |
| Returns: |
| A tuple of whether the collection succeeds and the generated trace file. |
| """ |
| tmp_output_file = tempfile.NamedTemporaryFile() |
| |
| collector = PerfettoTraceCollector(package=collector_info.package, |
| activity=None, |
| compiler_filter=collector_info.compiler_filter, |
| timeout=timeout, |
| simulate=simulate, |
| trace_duration=_PERFETTO_TRACE_DURATION, |
| save_destination_file_path=tmp_output_file.name) |
| result = collector.run() |
| |
| return result is not None, tmp_output_file |
| |
| def parse_run_script_csv_file(csv_file: TextIO) -> DataFrame: |
| """Parse a CSV file full of integers into a DataFrame.""" |
| csv_reader = csv.reader(csv_file) |
| |
| try: |
| header_list = next(csv_reader) |
| except StopIteration: |
| header_list = [] |
| |
| if not header_list: |
| return None |
| |
| headers = [i for i in header_list] |
| |
| d = {} |
| for row in csv_reader: |
| header_idx = 0 |
| |
| for i in row: |
| v = i |
| if i: |
| v = int(i) |
| |
| header_key = headers[header_idx] |
| l = d.get(header_key, []) |
| l.append(v) |
| d[header_key] = l |
| |
| header_idx = header_idx + 1 |
| |
| return DataFrame(d) |
| |
| def build_ri_compiler_argv(inodes_path: str, |
| perfetto_trace_file: str, |
| trace_duration: Optional[timedelta] |
| ) -> str: |
| argv = ['-i', inodes_path, '--perfetto-trace', |
| perfetto_trace_file] |
| |
| if trace_duration is not None: |
| argv += ['--duration', str(int(trace_duration.total_seconds() |
| * PerfettoTraceCollector.MS_PER_SEC))] |
| |
| print_utils.debug_print(argv) |
| return argv |
| |
| def execute_run_using_perfetto_trace(collector_info, |
| run_combos: Iterable[RunCommandArgs], |
| simulate: bool, |
| inodes_path: str, |
| timeout: int, |
| compiler_type: CompilerType, |
| requires_trace_collection: bool) -> DataFrame: |
| """ Executes run based on perfetto trace. """ |
| if requires_trace_collection: |
| passed, perfetto_trace_file = run_perfetto_collector(collector_info, |
| timeout, |
| simulate) |
| if not passed: |
| raise RuntimeError('Cannot run perfetto collector!') |
| else: |
| perfetto_trace_file = tempfile.NamedTemporaryFile() |
| |
| with perfetto_trace_file: |
| for combos in run_combos: |
| if combos.readahead in _TRACING_READAHEADS: |
| if simulate: |
| compiler_trace_file = tempfile.NamedTemporaryFile() |
| else: |
| ri_compiler_argv = build_ri_compiler_argv(inodes_path, |
| perfetto_trace_file.name, |
| combos.trace_duration) |
| compiler_trace_file = compiler.compile(compiler_type, |
| inodes_path, |
| ri_compiler_argv, |
| combos.package, |
| combos.activity) |
| |
| with compiler_trace_file: |
| combos = combos._replace(input=compiler_trace_file.name) |
| print_utils.debug_print(combos) |
| output = PrefetchAppRunner(**combos._asdict()).run() |
| else: |
| print_utils.debug_print(combos) |
| output = PrefetchAppRunner(**combos._asdict()).run() |
| |
| yield DataFrame(dict((x, [y]) for x, y in output)) if output else None |
| |
| def execute_run_combos( |
| grouped_run_combos: Iterable[Tuple[CollectorPackageInfo, Iterable[RunCommandArgs]]], |
| simulate: bool, |
| inodes_path: str, |
| timeout: int, |
| compiler_type: CompilerType, |
| requires_trace_collection: bool): |
| # nothing will work if the screen isn't unlocked first. |
| cmd_utils.execute_arbitrary_command([_UNLOCK_SCREEN_SCRIPT], |
| timeout, |
| simulate=simulate, |
| shell=False) |
| |
| for collector_info, run_combos in grouped_run_combos: |
| yield from execute_run_using_perfetto_trace(collector_info, |
| run_combos, |
| simulate, |
| inodes_path, |
| timeout, |
| compiler_type, |
| requires_trace_collection) |
| |
| def gather_results(commands: Iterable[Tuple[DataFrame]], |
| key_list: List[str], value_list: List[Tuple[str, ...]]): |
| print_utils.debug_print("gather_results: key_list = ", key_list) |
| stringify_none = lambda s: s is None and "<none>" or s |
| # yield key_list + ["time(ms)"] |
| for (run_result_list, values) in itertools.zip_longest(commands, value_list): |
| print_utils.debug_print("run_result_list = ", run_result_list) |
| print_utils.debug_print("values = ", values) |
| |
| if not run_result_list: |
| continue |
| |
| # RunCommandArgs(package='com.whatever', readahead='warm', compiler_filter=None) |
| # -> {'package':['com.whatever'], 'readahead':['warm'], 'compiler_filter':[None]} |
| values_dict = {} |
| for k, v in values._asdict().items(): |
| if not k in key_list: |
| continue |
| values_dict[k] = [stringify_none(v)] |
| |
| values_df = DataFrame(values_dict) |
| # project 'values_df' to be same number of rows as run_result_list. |
| values_df = values_df.repeat(run_result_list.data_row_len) |
| |
| # the results are added as right-hand-side columns onto the existing labels for the table. |
| values_df.merge_data_columns(run_result_list) |
| |
| yield values_df |
| |
| def eval_and_save_to_csv(output, annotated_result_values): |
| printed_header = False |
| |
| csv_writer = csv.writer(output) |
| for row in annotated_result_values: |
| if not printed_header: |
| headers = row.headers |
| csv_writer.writerow(headers) |
| printed_header = True |
| # TODO: what about when headers change? |
| |
| for data_row in row.data_table: |
| data_row = [d for d in data_row] |
| csv_writer.writerow(data_row) |
| |
| output.flush() # see the output live. |
| |
| def coerce_to_list(opts: dict): |
| """Tranform values of the dictionary to list. |
| For example: |
| 1 -> [1], None -> [None], [1,2,3] -> [1,2,3] |
| [[1],[2]] -> [[1],[2]], {1:1, 2:2} -> [{1:1, 2:2}] |
| """ |
| result = {} |
| for key in opts: |
| val = opts[key] |
| result[key] = val if issubclass(type(val), list) else [val] |
| return result |
| |
| def main(): |
| global _debug |
| |
| opts = parse_options() |
| _debug = opts.debug |
| if _DEBUG_FORCE is not None: |
| _debug = _DEBUG_FORCE |
| |
| print_utils.DEBUG = _debug |
| cmd_utils.SIMULATE = opts.simulate |
| |
| print_utils.debug_print("parsed options: ", opts) |
| |
| output_file = opts.output and open(opts.output, 'w') or sys.stdout |
| |
| combos = lambda: args_utils.generate_run_combinations( |
| RunCommandArgs, |
| coerce_to_list(vars(opts)), |
| opts.loop_count) |
| print_utils.debug_print_gen("run combinations: ", combos()) |
| |
| grouped_combos = lambda: args_utils.generate_group_run_combinations(combos(), |
| CollectorPackageInfo) |
| |
| print_utils.debug_print_gen("grouped run combinations: ", grouped_combos()) |
| requires_trace_collection = any(i in _TRACING_READAHEADS for i in opts.readaheads) |
| exec = execute_run_combos(grouped_combos(), |
| opts.simulate, |
| opts.inodes, |
| opts.timeout, |
| opts.compiler_type, |
| requires_trace_collection) |
| |
| results = gather_results(exec, _COMBINATORIAL_OPTIONS, combos()) |
| |
| eval_and_save_to_csv(output_file, results) |
| |
| return 1 |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |