startop/scripts/app_startup/app_startup_runner.py

#!/usr/bin/env python3
#
# Copyright 2018, The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
#
# Measure application start-up time by launching applications under various combinations.
# See --help for more details.
#
#
# Sample usage:
# $> ./app_startup_runner.py -p com.google.android.calculator -r warm -r cold -lc 10  -o out.csv
# $> ./analyze_metrics.py out.csv
#
#

import argparse
import csv
import itertools
import os
import sys
import tempfile
from datetime import timedelta
from typing import Any, Callable, Iterable, List, NamedTuple, TextIO, Tuple, \
    TypeVar, Union, Optional

# local import
DIR = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.dirname(DIR))
import lib.cmd_utils as cmd_utils
import lib.print_utils as print_utils
import iorap.compiler as compiler
from app_startup.run_app_with_prefetch import PrefetchAppRunner
import app_startup.lib.args_utils as args_utils
from app_startup.lib.data_frame import DataFrame
from app_startup.lib.perfetto_trace_collector import PerfettoTraceCollector

# The following command line options participate in the combinatorial generation.
# All other arguments have a global effect.
_COMBINATORIAL_OPTIONS = ['package', 'readahead', 'compiler_filter',
                          'activity', 'trace_duration']
_TRACING_READAHEADS = ['mlock', 'fadvise']
_FORWARD_OPTIONS = {'loop_count': '--count'}
_RUN_SCRIPT = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                           'run_app_with_prefetch.py')

CollectorPackageInfo = NamedTuple('CollectorPackageInfo',
                                  [('package', str), ('compiler_filter', str)])
_COMPILER_SCRIPT = os.path.join(os.path.dirname(os.path.dirname(
    os.path.realpath(__file__))), 'iorap/compiler.py')
# by 2; systrace starts up slowly.

_UNLOCK_SCREEN_SCRIPT = os.path.join(
    os.path.dirname(os.path.realpath(__file__)), 'unlock_screen')

RunCommandArgs = NamedTuple('RunCommandArgs',
                            [('package', str),
                             ('readahead', str),
                             ('activity', Optional[str]),
                             ('compiler_filter', Optional[str]),
                             ('timeout', Optional[int]),
                             ('debug', bool),
                             ('simulate', bool),
                             ('input', Optional[str]),
                             ('trace_duration', Optional[timedelta])])

# This must be the only mutable global variable. All other global variables are constants to avoid magic literals.
_debug = False  # See -d/--debug flag.
_DEBUG_FORCE = None  # Ignore -d/--debug if this is not none.
_PERFETTO_TRACE_DURATION_MS = 5000 # milliseconds
_PERFETTO_TRACE_DURATION = timedelta(milliseconds=_PERFETTO_TRACE_DURATION_MS)

# Type hinting names.
T = TypeVar('T')
NamedTupleMeta = Callable[
    ..., T]  # approximation of a (S : NamedTuple<T> where S() == T) metatype.

def parse_options(argv: List[str] = None):
  """Parse command line arguments and return an argparse Namespace object."""
  parser = argparse.ArgumentParser(description="Run one or more Android "
                                               "applications under various "
                                               "settings in order to measure "
                                               "startup time.")
  # argparse considers args starting with - and -- optional in --help, even though required=True.
  # by using a named argument group --help will clearly say that it's required instead of optional.
  required_named = parser.add_argument_group('required named arguments')
  required_named.add_argument('-p', '--package', action='append',
                              dest='packages',
                              help='package of the application', required=True)
  required_named.add_argument('-r', '--readahead', action='append',
                              dest='readaheads',
                              help='which readahead mode to use',
                              choices=('warm', 'cold', 'mlock', 'fadvise'),
                              required=True)

  # optional arguments
  # use a group here to get the required arguments to appear 'above' the optional arguments in help.
  optional_named = parser.add_argument_group('optional named arguments')
  optional_named.add_argument('-c', '--compiler-filter', action='append',
                              dest='compiler_filters',
                              help='which compiler filter to use. if omitted it does not enforce the app\'s compiler filter',
                              choices=('speed', 'speed-profile', 'quicken'))
  optional_named.add_argument('-s', '--simulate', dest='simulate',
                              action='store_true',
                              help='Print which commands will run, but don\'t run the apps')
  optional_named.add_argument('-d', '--debug', dest='debug',
                              action='store_true',
                              help='Add extra debugging output')
  optional_named.add_argument('-o', '--output', dest='output', action='store',
                              help='Write CSV output to file.')
  optional_named.add_argument('-t', '--timeout', dest='timeout', action='store',
                              type=int, default=10,
                              help='Timeout after this many seconds when executing a single run.')
  optional_named.add_argument('-lc', '--loop-count', dest='loop_count',
                              default=1, type=int, action='store',
                              help='How many times to loop a single run.')
  optional_named.add_argument('-in', '--inodes', dest='inodes', type=str,
                              action='store',
                              help='Path to inodes file (system/extras/pagecache/pagecache.py -d inodes)')
  optional_named.add_argument('--compiler-trace-duration-ms',
                              dest='trace_duration',
                              type=lambda ms_str: timedelta(milliseconds=int(ms_str)),
                              action='append',
                              help='The trace duration (milliseconds) in '
                                   'compilation')

  return parser.parse_args(argv)

def key_to_cmdline_flag(key: str) -> str:
  """Convert key into a command line flag, e.g. 'foo-bars' -> '--foo-bar' """
  if key.endswith("s"):
    key = key[:-1]
  return "--" + key.replace("_", "-")

def as_run_command(tpl: NamedTuple) -> List[Union[str, Any]]:
  """
  Convert a named tuple into a command-line compatible arguments list.

  Example: ABC(1, 2, 3) -> ['--a', 1, '--b', 2, '--c', 3]
  """
  args = []
  for key, value in tpl._asdict().items():
    if value is None:
      continue
    args.append(key_to_cmdline_flag(key))
    args.append(value)
  return args

def run_perfetto_collector(collector_info: CollectorPackageInfo,
                           timeout: int,
                           simulate: bool) -> Tuple[bool, TextIO]:
  """Run collector to collect prefetching trace.

  Returns:
    A tuple of whether the collection succeeds and the generated trace file.
  """
  tmp_output_file = tempfile.NamedTemporaryFile()

  collector = PerfettoTraceCollector(package=collector_info.package,
                                     activity=None,
                                     compiler_filter=collector_info.compiler_filter,
                                     timeout=timeout,
                                     simulate=simulate,
                                     trace_duration=_PERFETTO_TRACE_DURATION,
                                     save_destination_file_path=tmp_output_file.name)
  result = collector.run()

  return result is not None, tmp_output_file

def parse_run_script_csv_file(csv_file: TextIO) -> DataFrame:
  """Parse a CSV file full of integers into a DataFrame."""
  csv_reader = csv.reader(csv_file)

  try:
    header_list = next(csv_reader)
  except StopIteration:
    header_list = []

  if not header_list:
    return None

  headers = [i for i in header_list]

  d = {}
  for row in csv_reader:
    header_idx = 0

    for i in row:
      v = i
      if i:
        v = int(i)

      header_key = headers[header_idx]
      l = d.get(header_key, [])
      l.append(v)
      d[header_key] = l

      header_idx = header_idx + 1

  return DataFrame(d)

def compile_perfetto_trace(inodes_path: str,
                           perfetto_trace_file: str,
                           trace_duration: Optional[timedelta]) -> TextIO:
  compiler_trace_file = tempfile.NamedTemporaryFile()
  argv = [_COMPILER_SCRIPT, '-i', inodes_path, '--perfetto-trace',
          perfetto_trace_file, '-o', compiler_trace_file.name]

  if trace_duration is not None:
    argv += ['--duration', str(int(trace_duration.total_seconds()
                               * PerfettoTraceCollector.MS_PER_SEC))]

  print_utils.debug_print(argv)
  compiler.main(argv)
  return compiler_trace_file

def execute_run_using_perfetto_trace(collector_info,
                                     run_combos: Iterable[RunCommandArgs],
                                     simulate: bool,
                                     inodes_path: str,
                                     timeout: int) -> DataFrame:
  """ Executes run based on perfetto trace. """
  passed, perfetto_trace_file = run_perfetto_collector(collector_info,
                                                       timeout,
                                                       simulate)
  if not passed:
    raise RuntimeError('Cannot run perfetto collector!')

  with perfetto_trace_file:
    for combos in run_combos:
      if combos.readahead in _TRACING_READAHEADS:
        if simulate:
          compiler_trace_file = tempfile.NamedTemporaryFile()
        else:
          compiler_trace_file = compile_perfetto_trace(inodes_path,
                                                       perfetto_trace_file.name,
                                                       combos.trace_duration)
        with compiler_trace_file:
          combos = combos._replace(input=compiler_trace_file.name)
          print_utils.debug_print(combos)
          output = PrefetchAppRunner(**combos._asdict()).run()
      else:
        print_utils.debug_print(combos)
        output = PrefetchAppRunner(**combos._asdict()).run()

      yield DataFrame(dict((x, [y]) for x, y in output)) if output else None

def execute_run_combos(
    grouped_run_combos: Iterable[Tuple[CollectorPackageInfo, Iterable[RunCommandArgs]]],
    simulate: bool,
    inodes_path: str,
    timeout: int):
  # nothing will work if the screen isn't unlocked first.
  cmd_utils.execute_arbitrary_command([_UNLOCK_SCREEN_SCRIPT],
                                      timeout,
                                      simulate=simulate,
                                      shell=False)

  for collector_info, run_combos in grouped_run_combos:
    yield from execute_run_using_perfetto_trace(collector_info,
                                                run_combos,
                                                simulate,
                                                inodes_path,
                                                timeout)

def gather_results(commands: Iterable[Tuple[DataFrame]],
                   key_list: List[str], value_list: List[Tuple[str, ...]]):
  print_utils.debug_print("gather_results: key_list = ", key_list)
  stringify_none = lambda s: s is None and "<none>" or s
  #  yield key_list + ["time(ms)"]
  for (run_result_list, values) in itertools.zip_longest(commands, value_list):
    print_utils.debug_print("run_result_list = ", run_result_list)
    print_utils.debug_print("values = ", values)

    if not run_result_list:
      continue

    # RunCommandArgs(package='com.whatever', readahead='warm', compiler_filter=None)
    # -> {'package':['com.whatever'], 'readahead':['warm'], 'compiler_filter':[None]}
    values_dict = {}
    for k, v in values._asdict().items():
      if not k in key_list:
        continue
      values_dict[k] = [stringify_none(v)]

    values_df = DataFrame(values_dict)
    # project 'values_df' to be same number of rows as run_result_list.
    values_df = values_df.repeat(run_result_list.data_row_len)

    # the results are added as right-hand-side columns onto the existing labels for the table.
    values_df.merge_data_columns(run_result_list)

    yield values_df

def eval_and_save_to_csv(output, annotated_result_values):
  printed_header = False

  csv_writer = csv.writer(output)
  for row in annotated_result_values:
    if not printed_header:
      headers = row.headers
      csv_writer.writerow(headers)
      printed_header = True
      # TODO: what about when headers change?

    for data_row in row.data_table:
      data_row = [d for d in data_row]
      csv_writer.writerow(data_row)

    output.flush()  # see the output live.

def coerce_to_list(opts: dict):
  """Tranform values of the dictionary to list.
  For example:
  1 -> [1], None -> [None], [1,2,3] -> [1,2,3]
  [[1],[2]] -> [[1],[2]], {1:1, 2:2} -> [{1:1, 2:2}]
  """
  result = {}
  for key in opts:
    val = opts[key]
    result[key] = val if issubclass(type(val), list) else [val]
  return result

def main():
  global _debug

  opts = parse_options()
  _debug = opts.debug
  if _DEBUG_FORCE is not None:
    _debug = _DEBUG_FORCE

  print_utils.DEBUG = _debug
  cmd_utils.SIMULATE = opts.simulate

  print_utils.debug_print("parsed options: ", opts)

  output_file = opts.output and open(opts.output, 'w') or sys.stdout

  combos = lambda: args_utils.generate_run_combinations(
      RunCommandArgs,
      coerce_to_list(vars(opts)),
      opts.loop_count)
  print_utils.debug_print_gen("run combinations: ", combos())

  grouped_combos = lambda: args_utils.generate_group_run_combinations(combos(),
                                                                      CollectorPackageInfo)

  print_utils.debug_print_gen("grouped run combinations: ", grouped_combos())
  exec = execute_run_combos(grouped_combos(),
                            opts.simulate,
                            opts.inodes,
                            opts.timeout)

  results = gather_results(exec, _COMBINATORIAL_OPTIONS, combos())

  eval_and_save_to_csv(output_file, results)

  return 1

if __name__ == '__main__':
  sys.exit(main())