startop: add compiler type support to host and device switch.
When using device, the python modules for host are not loaded to be
compatiable in tradefed.
Test: pytest
Test: python app_startup_runner.py --package com.google.android.GoogleCamera --readahead fadvise --inodes textcache --output output.txt -lc 1 -d
Change-Id: I9fe54045fbabda5f66ebe1cfe6d0a5461fff0640
diff --git a/startop/scripts/app_startup/app_startup_runner.py b/startop/scripts/app_startup/app_startup_runner.py
index eb582f9..fa1c4e6 100755
--- a/startop/scripts/app_startup/app_startup_runner.py
+++ b/startop/scripts/app_startup/app_startup_runner.py
@@ -41,11 +41,12 @@
sys.path.append(os.path.dirname(DIR))
import lib.cmd_utils as cmd_utils
import lib.print_utils as print_utils
-import iorap.compiler as compiler
from app_startup.run_app_with_prefetch import PrefetchAppRunner
import app_startup.lib.args_utils as args_utils
from app_startup.lib.data_frame import DataFrame
from app_startup.lib.perfetto_trace_collector import PerfettoTraceCollector
+from iorap.compiler import CompilerType
+import iorap.compiler as compiler
# The following command line options participate in the combinatorial generation.
# All other arguments have a global effect.
@@ -58,8 +59,6 @@
CollectorPackageInfo = NamedTuple('CollectorPackageInfo',
[('package', str), ('compiler_filter', str)])
-_COMPILER_SCRIPT = os.path.join(os.path.dirname(os.path.dirname(
- os.path.realpath(__file__))), 'iorap/compiler.py')
# by 2; systrace starts up slowly.
_UNLOCK_SCREEN_SCRIPT = os.path.join(
@@ -135,6 +134,10 @@
action='append',
help='The trace duration (milliseconds) in '
'compilation')
+ optional_named.add_argument('--compiler-type', dest='compiler_type',
+ type=CompilerType, choices=list(CompilerType),
+ default=CompilerType.DEVICE,
+ help='The type of compiler.')
return parser.parse_args(argv)
@@ -211,26 +214,26 @@
return DataFrame(d)
-def compile_perfetto_trace(inodes_path: str,
+def build_ri_compiler_argv(inodes_path: str,
perfetto_trace_file: str,
- trace_duration: Optional[timedelta]) -> TextIO:
- compiler_trace_file = tempfile.NamedTemporaryFile()
- argv = [_COMPILER_SCRIPT, '-i', inodes_path, '--perfetto-trace',
- perfetto_trace_file, '-o', compiler_trace_file.name]
+ trace_duration: Optional[timedelta]
+ ) -> str:
+ argv = ['-i', inodes_path, '--perfetto-trace',
+ perfetto_trace_file]
if trace_duration is not None:
argv += ['--duration', str(int(trace_duration.total_seconds()
- * PerfettoTraceCollector.MS_PER_SEC))]
+ * PerfettoTraceCollector.MS_PER_SEC))]
print_utils.debug_print(argv)
- compiler.main(argv)
- return compiler_trace_file
+ return argv
def execute_run_using_perfetto_trace(collector_info,
run_combos: Iterable[RunCommandArgs],
simulate: bool,
inodes_path: str,
- timeout: int) -> DataFrame:
+ timeout: int,
+ compiler_type: CompilerType) -> DataFrame:
""" Executes run based on perfetto trace. """
passed, perfetto_trace_file = run_perfetto_collector(collector_info,
timeout,
@@ -244,9 +247,15 @@
if simulate:
compiler_trace_file = tempfile.NamedTemporaryFile()
else:
- compiler_trace_file = compile_perfetto_trace(inodes_path,
- perfetto_trace_file.name,
- combos.trace_duration)
+ ri_compiler_argv = build_ri_compiler_argv(inodes_path,
+ perfetto_trace_file.name,
+ combos.trace_duration)
+ compiler_trace_file = compiler.compile(compiler_type,
+ inodes_path,
+ ri_compiler_argv,
+ combos.package,
+ combos.activity)
+
with compiler_trace_file:
combos = combos._replace(input=compiler_trace_file.name)
print_utils.debug_print(combos)
@@ -261,7 +270,8 @@
grouped_run_combos: Iterable[Tuple[CollectorPackageInfo, Iterable[RunCommandArgs]]],
simulate: bool,
inodes_path: str,
- timeout: int):
+ timeout: int,
+ compiler_type: CompilerType):
# nothing will work if the screen isn't unlocked first.
cmd_utils.execute_arbitrary_command([_UNLOCK_SCREEN_SCRIPT],
timeout,
@@ -273,7 +283,8 @@
run_combos,
simulate,
inodes_path,
- timeout)
+ timeout,
+ compiler_type)
def gather_results(commands: Iterable[Tuple[DataFrame]],
key_list: List[str], value_list: List[Tuple[str, ...]]):
@@ -361,7 +372,8 @@
exec = execute_run_combos(grouped_combos(),
opts.simulate,
opts.inodes,
- opts.timeout)
+ opts.timeout,
+ opts.compiler_type)
results = gather_results(exec, _COMBINATORIAL_OPTIONS, combos())
diff --git a/startop/scripts/app_startup/app_startup_runner_test.py b/startop/scripts/app_startup/app_startup_runner_test.py
index 42ea5f0..382f6f3 100755
--- a/startop/scripts/app_startup/app_startup_runner_test.py
+++ b/startop/scripts/app_startup/app_startup_runner_test.py
@@ -92,7 +92,7 @@
"""
d = {'compiler_filters': None, 'simulate': False, 'debug': False,
'output': None, 'timeout': 10, 'loop_count': 1, 'inodes': None,
- 'trace_duration': None}
+ 'trace_duration': None, 'compiler_type': asr.CompilerType.HOST}
d.update(kwargs)
return d
diff --git a/startop/scripts/iorap/compiler.py b/startop/scripts/iorap/compiler.py
old mode 100755
new mode 100644
index 17b58c1..1426d34
--- a/startop/scripts/iorap/compiler.py
+++ b/startop/scripts/iorap/compiler.py
@@ -1,323 +1,73 @@
#!/usr/bin/env python3
-
#
-# Copyright (C) 2019 The Android Open Source Project
+# Copyright 2019, The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
-#
-# Dependencies:
-#
-# $> sudo apt-get install python3-pip
-# $> pip3 install --user protobuf sqlalchemy sqlite3
-#
-
-import optparse
+import importlib
import os
-import re
import sys
import tempfile
-from pathlib import Path
-from datetime import timedelta
-from typing import Iterable, Optional, List
+from enum import Enum
+from typing import TextIO, List
+# local import
DIR = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.dirname(DIR))
-from iorap.generated.TraceFile_pb2 import *
-from iorap.lib.inode2filename import Inode2Filename
+import lib.print_utils as print_utils
-parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-sys.path.append(parent_dir_name)
-from trace_analyzer.lib.trace2db import Trace2Db, MmFilemapAddToPageCache, \
- RawFtraceEntry
-import lib.cmd_utils as cmd_utils
+# Type of compiler.
+class CompilerType(Enum):
+ HOST = 1 # iorap.cmd.compiler on host
+ DEVICE = 2 # adb shell iorap.cmd.compiler
+ RI = 3 # compiler.py
-_PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes.
-ANDROID_BUILD_TOP = Path(parent_dir_name).parents[3]
-TRACECONV_BIN = ANDROID_BUILD_TOP.joinpath(
- 'external/perfetto/tools/traceconv')
+def compile_perfetto_trace_ri(
+ argv: List[str],
+ compiler) -> TextIO:
+ print_utils.debug_print('Compile using RI compiler.')
+ compiler_trace_file = tempfile.NamedTemporaryFile()
+ argv.extend(['-o', compiler_trace_file.name])
+ print_utils.debug_print(argv)
+ compiler.main([''] + argv)
+ return compiler_trace_file
-class PageRun:
- """
- Intermediate representation for a run of one or more pages.
- """
- def __init__(self, device_number: int, inode: int, offset: int, length: int):
- self.device_number = device_number
- self.inode = inode
- self.offset = offset
- self.length = length
+def compile_perfetto_trace_device(inodes_path: str,
+ package: str,
+ activity: str,
+ compiler) -> TextIO:
+ print_utils.debug_print('Compile using on-device compiler.')
+ compiler_trace_file = tempfile.NamedTemporaryFile()
+ compiler.main(inodes_path, package, activity, compiler_trace_file.name)
+ return compiler_trace_file
- def __str__(self):
- return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \
- %(self.device_number, self.inode, self.offset, self.length)
+def compile(compiler_type: CompilerType,
+ inodes_path: str,
+ ri_compiler_argv,
+ package: str,
+ activity: str) -> TextIO:
+ if compiler_type == CompilerType.RI:
+ compiler = importlib.import_module('iorap.compiler_ri')
+ compiler_trace_file = compile_perfetto_trace_ri(ri_compiler_argv,
+ compiler)
+ return compiler_trace_file
+ if compiler_type == CompilerType.DEVICE:
+ compiler = importlib.import_module('iorap.compiler_device')
+ compiler_trace_file = compile_perfetto_trace_device(inodes_path,
+ package,
+ activity,
+ compiler)
+ return compiler_trace_file
-def debug_print(msg):
- #print(msg)
- pass
-
-UNDER_LAUNCH = False
-
-def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]):
- global _PAGE_SIZE
-
- runs = [
- PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs,
- length=_PAGE_SIZE)
- for pg_entry in page_cache_entries
- ]
-
- for r in runs:
- debug_print(r)
-
- print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE))
-
- return runs
-
-def optimize_page_runs(page_runs):
- new_entries = []
- last_entry = None
- for pg_entry in page_runs:
- if last_entry:
- if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode:
- # we are dealing with a run for the same exact file as a previous run.
- if pg_entry.offset == last_entry.offset + last_entry.length:
- # trivially contiguous entries. merge them together.
- last_entry.length += pg_entry.length
- continue
- # Default: Add the run without merging it to a previous run.
- last_entry = pg_entry
- new_entries.append(pg_entry)
- return new_entries
-
-def is_filename_matching_filter(file_name, filters=[]):
- """
- Blacklist-style regular expression filters.
-
- :return: True iff file_name has an RE match in one of the filters.
- """
- for filt in filters:
- res = re.search(filt, file_name)
- if res:
- return True
-
- return False
-
-def build_protobuf(page_runs, inode2filename, filters=[]):
- trace_file = TraceFile()
- trace_file_index = trace_file.index
-
- file_id_counter = 0
- file_id_map = {} # filename -> id
-
- stats_length_total = 0
- filename_stats = {} # filename -> total size
-
- skipped_inode_map = {}
- filtered_entry_map = {} # filename -> count
-
- for pg_entry in page_runs:
- fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode)
- if not fn:
- skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1
- continue
-
- filename = fn
-
- if filters and not is_filename_matching_filter(filename, filters):
- filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1
- continue
-
- file_id = file_id_map.get(filename)
- if not file_id:
- file_id = file_id_counter
- file_id_map[filename] = file_id_counter
- file_id_counter = file_id_counter + 1
-
- file_index_entry = trace_file_index.entries.add()
- file_index_entry.id = file_id
- file_index_entry.file_name = filename
-
- # already in the file index, add the file entry.
- file_entry = trace_file.list.entries.add()
- file_entry.index_id = file_id
- file_entry.file_length = pg_entry.length
- stats_length_total += file_entry.file_length
- file_entry.file_offset = pg_entry.offset
-
- filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length
-
- for inode, count in skipped_inode_map.items():
- print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count))
-
- print("Stats: Sum of lengths %d" %(stats_length_total))
-
- if filters:
- print("Filter: %d total files removed." %(len(filtered_entry_map)))
-
- for fn, count in filtered_entry_map.items():
- print("Filter: File '%s' removed '%d' entries." %(fn, count))
-
- for filename, file_size in filename_stats.items():
- print("%s,%s" %(filename, file_size))
-
- return trace_file
-
-def calc_trace_end_time(trace2db: Trace2Db,
- trace_duration: Optional[timedelta]) -> float:
- """
- Calculates the end time based on the trace duration.
- The start time is the first receiving mm file map event.
- The end time is the start time plus the trace duration.
- All of them are in milliseconds.
- """
- # If the duration is not set, assume all time is acceptable.
- if trace_duration is None:
- # float('inf')
- return RawFtraceEntry.__table__.c.timestamp.type.python_type('inf')
-
- first_event = trace2db.session.query(MmFilemapAddToPageCache).join(
- MmFilemapAddToPageCache.raw_ftrace_entry).order_by(
- RawFtraceEntry.timestamp).first()
-
- # total_seconds() will return a float number.
- return first_event.raw_ftrace_entry.timestamp + trace_duration.total_seconds()
-
-def query_add_to_page_cache(trace2db: Trace2Db, trace_duration: Optional[timedelta]):
- end_time = calc_trace_end_time(trace2db, trace_duration)
- # SELECT * FROM tbl ORDER BY id;
- return trace2db.session.query(MmFilemapAddToPageCache).join(
- MmFilemapAddToPageCache.raw_ftrace_entry).filter(
- RawFtraceEntry.timestamp <= end_time).order_by(
- MmFilemapAddToPageCache.id).all()
-
-def transform_perfetto_trace_to_systrace(path_to_perfetto_trace: str,
- path_to_tmp_systrace: str) -> None:
- """ Transforms the systrace file from perfetto trace. """
- cmd_utils.run_command_nofail([str(TRACECONV_BIN),
- 'systrace',
- path_to_perfetto_trace,
- path_to_tmp_systrace])
-
-
-def run(sql_db_path:str,
- trace_file:str,
- trace_duration:Optional[timedelta],
- output_file:str,
- inode_table:str,
- filter:List[str]) -> int:
- trace2db = Trace2Db(sql_db_path)
- # Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache.
- trace2db.set_raw_ftrace_entry_filter(\
- lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache')
- # TODO: parse multiple trace files here.
- parse_count = trace2db.parse_file_into_db(trace_file)
-
- mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db,
- trace_duration)
- print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows)))
-
- page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows)
- print("DONE. Converted %d entries" %(len(page_runs)))
-
- # TODO: flags to select optimizations.
- optimized_page_runs = optimize_page_runs(page_runs)
- print("DONE. Optimized down to %d entries" %(len(optimized_page_runs)))
-
- print("Build protobuf...")
- trace_file = build_protobuf(optimized_page_runs, inode_table, filter)
-
- print("Write protobuf to file...")
- output_file = open(output_file, 'wb')
- output_file.write(trace_file.SerializeToString())
- output_file.close()
-
- print("DONE")
-
- # TODO: Silent running mode [no output except on error] for build runs.
-
- return 0
-
-def main(argv):
- parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb")
- parser.add_option('-i', dest='inode_data_file', metavar='FILE',
- help='Read cached inode data from a file saved earlier with pagecache.py -d')
- parser.add_option('-t', dest='trace_file', metavar='FILE',
- help='Path to systrace file (trace.html) that will be parsed')
- parser.add_option('--perfetto-trace', dest='perfetto_trace_file',
- metavar='FILE',
- help='Path to perfetto trace that will be parsed')
-
- parser.add_option('--db', dest='sql_db', metavar='FILE',
- help='Path to intermediate sqlite3 database [default: in-memory].')
-
- parser.add_option('-f', dest='filter', action="append", default=[],
- help="Add file filter. All file entries not matching one of the filters are discarded.")
-
- parser.add_option('-l', dest='launch_lock', action="store_true", default=False,
- help="Exclude all events not inside launch_lock")
-
- parser.add_option('-o', dest='output_file', metavar='FILE',
- help='Output protobuf file')
-
- parser.add_option('--duration', dest='trace_duration', action="store",
- type=int, help='The duration of trace in milliseconds.')
-
- options, categories = parser.parse_args(argv[1:])
-
- # TODO: OptionParser should have some flags to make these mandatory.
- if not options.inode_data_file:
- parser.error("-i is required")
- if not options.trace_file and not options.perfetto_trace_file:
- parser.error("one of -t or --perfetto-trace is required")
- if options.trace_file and options.perfetto_trace_file:
- parser.error("please enter either -t or --perfetto-trace, not both")
- if not options.output_file:
- parser.error("-o is required")
-
- if options.launch_lock:
- print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.")
-
- inode_table = Inode2Filename.new_from_filename(options.inode_data_file)
-
- sql_db_path = ":memory:"
- if options.sql_db:
- sql_db_path = options.sql_db
-
- trace_duration = timedelta(milliseconds=options.trace_duration) if \
- options.trace_duration is not None else None
-
- # if the input is systrace
- if options.trace_file:
- return run(sql_db_path,
- options.trace_file,
- trace_duration,
- options.output_file,
- inode_table,
- options.filter)
-
- # if the input is perfetto trace
- # TODO python 3.7 switch to using nullcontext
- with tempfile.NamedTemporaryFile() as trace_file:
- transform_perfetto_trace_to_systrace(options.perfetto_trace_file,
- trace_file.name)
- return run(sql_db_path,
- trace_file.name,
- trace_duration,
- options.output_file,
- inode_table,
- options.filter)
-
-if __name__ == '__main__':
- print(sys.argv)
- sys.exit(main(sys.argv))
+ # Should not arrive here.
+ raise ValueError('Unknown compiler type')
diff --git a/startop/scripts/iorap/compiler_device.py b/startop/scripts/iorap/compiler_device.py
new file mode 100644
index 0000000..d941cd9
--- /dev/null
+++ b/startop/scripts/iorap/compiler_device.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import sys
+from typing import List
+
+DIR = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.dirname(DIR)) # framework/base/startop/script
+import lib.print_utils as print_utils
+import iorap.lib.iorapd_utils as iorapd_utils
+from app_startup.lib.app_runner import AppRunner
+
+IORAP_COMMON_BASH_SCRIPT = os.path.join(DIR, 'common')
+
+def parse_options(argv: List[str] = None):
+ """Parses command line arguments and returns an argparse Namespace object."""
+ parser = argparse.ArgumentParser(description="Compile perfetto trace file")
+ required_named = parser.add_argument_group('required named arguments')
+
+ required_named.add_argument('-i', dest='inodes', metavar='FILE',
+ help='Read cached inode data from a file saved '
+ 'earlier with pagecache.py -d')
+ required_named.add_argument('-p', dest='package',
+ help='Package of the app to be compiled')
+
+ optional_named = parser.add_argument_group('optional named arguments')
+ optional_named.add_argument('-o', dest='output',
+ help='The compiled trace is stored into the output file')
+ optional_named.add_argument('-a', dest='activity',
+ help='Activity of the app to be compiled')
+ optional_named.add_argument('-d', dest='debug', action='store_true'
+ , help='Activity of the app to be compiled')
+
+ return parser.parse_args(argv)
+
+def main(inodes, package, activity, output, **kwargs) -> int:
+ """Entries of the program."""
+ if not activity:
+ activity = AppRunner.get_activity(package)
+
+ passed = iorapd_utils.compile_perfetto_trace_on_device(package, activity,
+ inodes)
+ if passed and output:
+ iorapd_utils.get_iorapd_compiler_trace(package, activity, output)
+
+ return 0
+
+if __name__ == '__main__':
+ opts = parse_options()
+ if opts.debug:
+ print_utils.DEBUG = opts.debug
+ print_utils.debug_print(opts)
+ sys.exit(main(**(vars(opts))))
diff --git a/startop/scripts/iorap/compiler_ri.py b/startop/scripts/iorap/compiler_ri.py
new file mode 100755
index 0000000..17b58c1
--- /dev/null
+++ b/startop/scripts/iorap/compiler_ri.py
@@ -0,0 +1,323 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Dependencies:
+#
+# $> sudo apt-get install python3-pip
+# $> pip3 install --user protobuf sqlalchemy sqlite3
+#
+
+import optparse
+import os
+import re
+import sys
+import tempfile
+from pathlib import Path
+from datetime import timedelta
+from typing import Iterable, Optional, List
+
+DIR = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.dirname(DIR))
+from iorap.generated.TraceFile_pb2 import *
+from iorap.lib.inode2filename import Inode2Filename
+
+parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+sys.path.append(parent_dir_name)
+from trace_analyzer.lib.trace2db import Trace2Db, MmFilemapAddToPageCache, \
+ RawFtraceEntry
+import lib.cmd_utils as cmd_utils
+
+_PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes.
+ANDROID_BUILD_TOP = Path(parent_dir_name).parents[3]
+TRACECONV_BIN = ANDROID_BUILD_TOP.joinpath(
+ 'external/perfetto/tools/traceconv')
+
+class PageRun:
+ """
+ Intermediate representation for a run of one or more pages.
+ """
+ def __init__(self, device_number: int, inode: int, offset: int, length: int):
+ self.device_number = device_number
+ self.inode = inode
+ self.offset = offset
+ self.length = length
+
+ def __str__(self):
+ return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \
+ %(self.device_number, self.inode, self.offset, self.length)
+
+def debug_print(msg):
+ #print(msg)
+ pass
+
+UNDER_LAUNCH = False
+
+def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]):
+ global _PAGE_SIZE
+
+ runs = [
+ PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs,
+ length=_PAGE_SIZE)
+ for pg_entry in page_cache_entries
+ ]
+
+ for r in runs:
+ debug_print(r)
+
+ print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE))
+
+ return runs
+
+def optimize_page_runs(page_runs):
+ new_entries = []
+ last_entry = None
+ for pg_entry in page_runs:
+ if last_entry:
+ if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode:
+ # we are dealing with a run for the same exact file as a previous run.
+ if pg_entry.offset == last_entry.offset + last_entry.length:
+ # trivially contiguous entries. merge them together.
+ last_entry.length += pg_entry.length
+ continue
+ # Default: Add the run without merging it to a previous run.
+ last_entry = pg_entry
+ new_entries.append(pg_entry)
+ return new_entries
+
+def is_filename_matching_filter(file_name, filters=[]):
+ """
+ Blacklist-style regular expression filters.
+
+ :return: True iff file_name has an RE match in one of the filters.
+ """
+ for filt in filters:
+ res = re.search(filt, file_name)
+ if res:
+ return True
+
+ return False
+
+def build_protobuf(page_runs, inode2filename, filters=[]):
+ trace_file = TraceFile()
+ trace_file_index = trace_file.index
+
+ file_id_counter = 0
+ file_id_map = {} # filename -> id
+
+ stats_length_total = 0
+ filename_stats = {} # filename -> total size
+
+ skipped_inode_map = {}
+ filtered_entry_map = {} # filename -> count
+
+ for pg_entry in page_runs:
+ fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode)
+ if not fn:
+ skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1
+ continue
+
+ filename = fn
+
+ if filters and not is_filename_matching_filter(filename, filters):
+ filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1
+ continue
+
+ file_id = file_id_map.get(filename)
+ if not file_id:
+ file_id = file_id_counter
+ file_id_map[filename] = file_id_counter
+ file_id_counter = file_id_counter + 1
+
+ file_index_entry = trace_file_index.entries.add()
+ file_index_entry.id = file_id
+ file_index_entry.file_name = filename
+
+ # already in the file index, add the file entry.
+ file_entry = trace_file.list.entries.add()
+ file_entry.index_id = file_id
+ file_entry.file_length = pg_entry.length
+ stats_length_total += file_entry.file_length
+ file_entry.file_offset = pg_entry.offset
+
+ filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length
+
+ for inode, count in skipped_inode_map.items():
+ print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count))
+
+ print("Stats: Sum of lengths %d" %(stats_length_total))
+
+ if filters:
+ print("Filter: %d total files removed." %(len(filtered_entry_map)))
+
+ for fn, count in filtered_entry_map.items():
+ print("Filter: File '%s' removed '%d' entries." %(fn, count))
+
+ for filename, file_size in filename_stats.items():
+ print("%s,%s" %(filename, file_size))
+
+ return trace_file
+
+def calc_trace_end_time(trace2db: Trace2Db,
+ trace_duration: Optional[timedelta]) -> float:
+ """
+ Calculates the end time based on the trace duration.
+ The start time is the first receiving mm file map event.
+ The end time is the start time plus the trace duration.
+ All of them are in milliseconds.
+ """
+ # If the duration is not set, assume all time is acceptable.
+ if trace_duration is None:
+ # float('inf')
+ return RawFtraceEntry.__table__.c.timestamp.type.python_type('inf')
+
+ first_event = trace2db.session.query(MmFilemapAddToPageCache).join(
+ MmFilemapAddToPageCache.raw_ftrace_entry).order_by(
+ RawFtraceEntry.timestamp).first()
+
+ # total_seconds() will return a float number.
+ return first_event.raw_ftrace_entry.timestamp + trace_duration.total_seconds()
+
+def query_add_to_page_cache(trace2db: Trace2Db, trace_duration: Optional[timedelta]):
+ end_time = calc_trace_end_time(trace2db, trace_duration)
+ # SELECT * FROM tbl ORDER BY id;
+ return trace2db.session.query(MmFilemapAddToPageCache).join(
+ MmFilemapAddToPageCache.raw_ftrace_entry).filter(
+ RawFtraceEntry.timestamp <= end_time).order_by(
+ MmFilemapAddToPageCache.id).all()
+
+def transform_perfetto_trace_to_systrace(path_to_perfetto_trace: str,
+ path_to_tmp_systrace: str) -> None:
+ """ Transforms the systrace file from perfetto trace. """
+ cmd_utils.run_command_nofail([str(TRACECONV_BIN),
+ 'systrace',
+ path_to_perfetto_trace,
+ path_to_tmp_systrace])
+
+
+def run(sql_db_path:str,
+ trace_file:str,
+ trace_duration:Optional[timedelta],
+ output_file:str,
+ inode_table:str,
+ filter:List[str]) -> int:
+ trace2db = Trace2Db(sql_db_path)
+ # Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache.
+ trace2db.set_raw_ftrace_entry_filter(\
+ lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache')
+ # TODO: parse multiple trace files here.
+ parse_count = trace2db.parse_file_into_db(trace_file)
+
+ mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db,
+ trace_duration)
+ print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows)))
+
+ page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows)
+ print("DONE. Converted %d entries" %(len(page_runs)))
+
+ # TODO: flags to select optimizations.
+ optimized_page_runs = optimize_page_runs(page_runs)
+ print("DONE. Optimized down to %d entries" %(len(optimized_page_runs)))
+
+ print("Build protobuf...")
+ trace_file = build_protobuf(optimized_page_runs, inode_table, filter)
+
+ print("Write protobuf to file...")
+ output_file = open(output_file, 'wb')
+ output_file.write(trace_file.SerializeToString())
+ output_file.close()
+
+ print("DONE")
+
+ # TODO: Silent running mode [no output except on error] for build runs.
+
+ return 0
+
+def main(argv):
+ parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb")
+ parser.add_option('-i', dest='inode_data_file', metavar='FILE',
+ help='Read cached inode data from a file saved earlier with pagecache.py -d')
+ parser.add_option('-t', dest='trace_file', metavar='FILE',
+ help='Path to systrace file (trace.html) that will be parsed')
+ parser.add_option('--perfetto-trace', dest='perfetto_trace_file',
+ metavar='FILE',
+ help='Path to perfetto trace that will be parsed')
+
+ parser.add_option('--db', dest='sql_db', metavar='FILE',
+ help='Path to intermediate sqlite3 database [default: in-memory].')
+
+ parser.add_option('-f', dest='filter', action="append", default=[],
+ help="Add file filter. All file entries not matching one of the filters are discarded.")
+
+ parser.add_option('-l', dest='launch_lock', action="store_true", default=False,
+ help="Exclude all events not inside launch_lock")
+
+ parser.add_option('-o', dest='output_file', metavar='FILE',
+ help='Output protobuf file')
+
+ parser.add_option('--duration', dest='trace_duration', action="store",
+ type=int, help='The duration of trace in milliseconds.')
+
+ options, categories = parser.parse_args(argv[1:])
+
+ # TODO: OptionParser should have some flags to make these mandatory.
+ if not options.inode_data_file:
+ parser.error("-i is required")
+ if not options.trace_file and not options.perfetto_trace_file:
+ parser.error("one of -t or --perfetto-trace is required")
+ if options.trace_file and options.perfetto_trace_file:
+ parser.error("please enter either -t or --perfetto-trace, not both")
+ if not options.output_file:
+ parser.error("-o is required")
+
+ if options.launch_lock:
+ print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.")
+
+ inode_table = Inode2Filename.new_from_filename(options.inode_data_file)
+
+ sql_db_path = ":memory:"
+ if options.sql_db:
+ sql_db_path = options.sql_db
+
+ trace_duration = timedelta(milliseconds=options.trace_duration) if \
+ options.trace_duration is not None else None
+
+ # if the input is systrace
+ if options.trace_file:
+ return run(sql_db_path,
+ options.trace_file,
+ trace_duration,
+ options.output_file,
+ inode_table,
+ options.filter)
+
+ # if the input is perfetto trace
+ # TODO python 3.7 switch to using nullcontext
+ with tempfile.NamedTemporaryFile() as trace_file:
+ transform_perfetto_trace_to_systrace(options.perfetto_trace_file,
+ trace_file.name)
+ return run(sql_db_path,
+ trace_file.name,
+ trace_duration,
+ options.output_file,
+ inode_table,
+ options.filter)
+
+if __name__ == '__main__':
+ print(sys.argv)
+ sys.exit(main(sys.argv))
diff --git a/startop/scripts/iorap/compiler_test.py b/startop/scripts/iorap/compiler_test.py
index 1a9f059..d1f11c5 100644
--- a/startop/scripts/iorap/compiler_test.py
+++ b/startop/scripts/iorap/compiler_test.py
@@ -30,7 +30,7 @@
"""
import os
-import compiler
+import compiler_host as compiler
DIR = os.path.abspath(os.path.dirname(__file__))
TEXTCACHE = os.path.join(DIR, 'test_fixtures/compiler/common_textcache')
diff --git a/startop/scripts/iorap/lib/iorapd_utils.py b/startop/scripts/iorap/lib/iorapd_utils.py
index 0d62180..f6f21fd 100644
--- a/startop/scripts/iorap/lib/iorapd_utils.py
+++ b/startop/scripts/iorap/lib/iorapd_utils.py
@@ -18,10 +18,9 @@
import os
import sys
-from pathlib import Path
-# up to two level, like '../../'
-sys.path.append(Path(os.path.abspath(__file__)).parents[2])
+# up to two level
+sys.path.append(os.path.join(os.path.abspath(__file__),'../..'))
import lib.cmd_utils as cmd_utils
IORAPID_LIB_DIR = os.path.abspath(os.path.dirname(__file__))
@@ -39,6 +38,22 @@
# Match logic of 'AppComponentName' in iorap::compiler C++ code.
return '{}/{}%2F{}.{}'.format(IORAPD_DATA_PATH, package, activity, suffix)
+def compile_perfetto_trace_on_device(package: str, activity: str,
+ inodes: str) -> bool:
+ """Compiles the perfetto trace using on-device compiler."""
+ passed, _ = cmd_utils.run_shell_func(IORAP_COMMON_BASH_SCRIPT,
+ 'iorapd_compiler_for_app_trace',
+ [package, activity, inodes])
+ return passed
+
+def get_iorapd_compiler_trace(package: str, activity: str, dest: str) -> str:
+ """Gets compiler trace to dest file."""
+ src = _iorapd_path_to_data_file(package, activity, 'compiled_trace.pb')
+ passed, _ = cmd_utils.run_shell_command('adb pull "{}" "{}"'.format(src, dest))
+ if not passed:
+ return False
+ return True
+
def iorapd_compiler_install_trace_file(package: str, activity: str,
input_file: str) -> bool:
"""Installs a compiled trace file.