startop: Update app_startup_runner to work with iorapd
Script measures app startup times with/without iorapd-based
prefetching against a cold (or warm) baseline.
mlock/fadvise readahead options now use iorapd to exercise
real code to do prefetching.
Bug: 72170747
Change-Id: I28affac3470271e46febb0e22a6dc9543258b4a4
diff --git a/startop/iorap/tests/src/com/google/android/startop/iorap/IIorapIntegrationTest.kt b/startop/iorap/tests/src/com/google/android/startop/iorap/IIorapIntegrationTest.kt
index b1e6194..883d094 100644
--- a/startop/iorap/tests/src/com/google/android/startop/iorap/IIorapIntegrationTest.kt
+++ b/startop/iorap/tests/src/com/google/android/startop/iorap/IIorapIntegrationTest.kt
@@ -105,6 +105,13 @@
}
@Test
+ fun testOnAppLaunchEvent() {
+ testAnyMethod { requestId : RequestId ->
+ // iorapService.onAppLaunchEvent(requestId, AppLaunchEvent.IntentStarted())
+ }
+ }
+
+ @Test
fun testOnSystemServiceEvent() {
testAnyMethod { requestId: RequestId ->
iorapService.onSystemServiceEvent(requestId,
diff --git a/startop/scripts/app_startup/app_startup_runner.py b/startop/scripts/app_startup/app_startup_runner.py
index 780bb4e..adf2ad8 100755
--- a/startop/scripts/app_startup/app_startup_runner.py
+++ b/startop/scripts/app_startup/app_startup_runner.py
@@ -27,12 +27,13 @@
#
import argparse
+import asyncio
import csv
import itertools
import os
-import subprocess
import sys
import tempfile
+import time
from typing import Any, Callable, Dict, Generic, Iterable, List, NamedTuple, TextIO, Tuple, TypeVar, Optional, Union
# The following command line options participate in the combinatorial generation.
@@ -44,7 +45,7 @@
RunCommandArgs = NamedTuple('RunCommandArgs', [('package', str), ('readahead', str), ('compiler_filter', Optional[str])])
CollectorPackageInfo = NamedTuple('CollectorPackageInfo', [('package', str), ('compiler_filter', str)])
-_COLLECTOR_SCRIPT=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'collector')
+_COLLECTOR_SCRIPT=os.path.join(os.path.dirname(os.path.realpath(__file__)), '../iorap/collector')
_COLLECTOR_TIMEOUT_MULTIPLIER = 2 # take the regular --timeout and multiply by 2; systrace starts up slowly.
_UNLOCK_SCREEN_SCRIPT=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'unlock_screen')
@@ -217,25 +218,95 @@
cmd = cmd + ["--output", tmp_output_file.name]
return cmd, tmp_output_file
-def execute_arbitrary_command(cmd: List[str], simulate: bool, timeout: int) -> Tuple[bool, str]:
+async def _run_command(*args : List[str], timeout: Optional[int] = None) -> Tuple[int, bytes]:
+ # start child process
+ # NOTE: universal_newlines parameter is not supported
+ process = await asyncio.create_subprocess_exec(*args,
+ stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT)
+
+ script_output = b""
+
+ _debug_print("[PID]", process.pid)
+
+#hack
+# stdout, stderr = await process.communicate()
+# return (process.returncode, stdout)
+
+ timeout_remaining = timeout
+ time_started = time.time()
+
+ # read line (sequence of bytes ending with b'\n') asynchronously
+ while True:
+ try:
+ line = await asyncio.wait_for(process.stdout.readline(), timeout_remaining)
+ _debug_print("[STDOUT]", line)
+ script_output += line
+
+ if timeout_remaining:
+ time_elapsed = time.time() - time_started
+ timeout_remaining = timeout - time_elapsed
+ except asyncio.TimeoutError:
+ _debug_print("[TIMEDOUT] Process ", process.pid)
+
+# if process.returncode is not None:
+# #_debug_print("[WTF] can-write-eof?", process.stdout.can_write_eof())
+#
+# _debug_print("[TIMEDOUT] Process already terminated")
+# (remaining_stdout, remaining_stderr) = await process.communicate()
+# script_output += remaining_stdout
+#
+# code = await process.wait()
+# return (code, script_output)
+
+ _debug_print("[TIMEDOUT] Sending SIGTERM.")
+ process.terminate()
+
+ # 1 second timeout for process to handle SIGTERM nicely.
+ try:
+ (remaining_stdout, remaining_stderr) = await asyncio.wait_for(process.communicate(), 5)
+ script_output += remaining_stdout
+ except asyncio.TimeoutError:
+ _debug_print("[TIMEDOUT] Sending SIGKILL.")
+ process.kill()
+
+ # 1 second timeout to finish with SIGKILL.
+ try:
+ (remaining_stdout, remaining_stderr) = await asyncio.wait_for(process.communicate(), 5)
+ script_output += remaining_stdout
+ except asyncio.TimeoutError:
+ # give up, this will leave a zombie process.
+ _debug_print("[TIMEDOUT] SIGKILL failed for process ", process.pid)
+ time.sleep(100)
+ #await process.wait()
+
+ return (-1, script_output)
+ else:
+ if not line: # EOF
+ break
+
+ #if process.returncode is not None:
+ # _debug_print("[WTF] can-write-eof?", process.stdout.can_write_eof())
+ # process.stdout.write_eof()
+
+ #if process.stdout.at_eof():
+ # break
+
+ code = await process.wait() # wait for child process to exit
+ return (code, script_output)
+
+def execute_arbitrary_command(cmd: List[str], simulate: bool, timeout: Optional[int]) -> Tuple[bool, str]:
if simulate:
print(" ".join(cmd))
return (True, "")
else:
_debug_print("[EXECUTE]", cmd)
- proc = subprocess.Popen(cmd,
- stderr=subprocess.STDOUT,
- stdout=subprocess.PIPE,
- universal_newlines=True)
- try:
- script_output = proc.communicate(timeout=timeout)[0]
- except subprocess.TimeoutExpired:
- print("[TIMEDOUT]")
- proc.kill()
- script_output = proc.communicate()[0]
- _debug_print("[STDOUT]", script_output)
- return_code = proc.wait()
+ # block until either command finishes or the timeout occurs.
+ loop = asyncio.get_event_loop()
+ (return_code, script_output) = loop.run_until_complete(_run_command(*cmd, timeout=timeout))
+
+ script_output = script_output.decode() # convert bytes to str
+
passed = (return_code == 0)
_debug_print("[$?]", return_code)
if not passed:
diff --git a/startop/scripts/app_startup/launch_application b/startop/scripts/app_startup/launch_application
index 8a68e50..bc7ca80 100755
--- a/startop/scripts/app_startup/launch_application
+++ b/startop/scripts/app_startup/launch_application
@@ -17,6 +17,19 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "$DIR/lib/common"
+launch_application_usage() {
+ cat <<EOF
+Usage: $(basename $0) <package> <activity>
+
+ Positional Arguments:
+ <package> package of the app to test
+ <activity> activity to use
+
+ Named Arguments:
+ -h, --help usage information (this)
+EOF
+}
+
launch_application() {
local package="$1"
local activity="$2"
@@ -44,4 +57,9 @@
echo "$total_time"
}
+if [[ $# -lt 2 ]]; then
+ launch_application_usage
+ exit 1
+fi
+
launch_application "$@"
diff --git a/startop/scripts/app_startup/lib/common b/startop/scripts/app_startup/lib/common
index 043d855..41f8cda 100755
--- a/startop/scripts/app_startup/lib/common
+++ b/startop/scripts/app_startup/lib/common
@@ -1,4 +1,17 @@
#!/bin/bash
+# Copyright 2018, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
if [[ -z $ANDROID_BUILD_TOP ]]; then
echo "Please run source build/envsetup.sh first" >&2
@@ -49,5 +62,103 @@
local activity_line="$(adb shell cmd package query-activities --brief -a android.intent.action.MAIN -c android.intent.category.LAUNCHER | grep "$package/")"
IFS="/" read -a array <<< "$activity_line"
local activity_name="${array[1]}"
+
+ # Activities starting with '.' are shorthand for having their package name prefixed.
+ if [[ $activity_name == .* ]]; then
+ activity_name="${package}${activity_name}"
+ fi
echo "$activity_name"
}
+
+# Use with logcat_from_timestamp to skip all past log-lines.
+logcat_save_timestamp() {
+ adb shell 'date -u +"%Y-%m-%d %H:%M:%S.%N"'
+}
+
+# Roll forward logcat to only show events
+# since the specified timestamp.
+#
+# i.e. don't look at historical logcat,
+# only look at FUTURE logcat.
+#
+# First use 'logcat_save_timestamp'
+# Then do whatever action you want.
+# Then us 'logcat_from_timestamp $timestamp'
+logcat_from_timestamp() {
+ local timestamp="$1"
+ shift # drop timestamp from args.
+ echo "DONT CALL THIS FUNCTION" >&2
+ exit 1
+
+ verbose_print adb logcat -T \"$timestamp\" \"$@\"
+ adb logcat -T "$timestamp" "$@"
+}
+
+logcat_from_timestamp_bg() {
+ local timestamp="$1"
+ shift # drop timestamp from args.
+ verbose_print adb logcat -T \"$timestamp\" \"$@\"
+ adb logcat -v UTC -T "$timestamp" "$@" &
+ logcat_from_timestamp_pid=$!
+}
+
+# Starting at timestamp $2, wait until we seen pattern $3
+# or until a timeout happens in $1 seconds.
+#
+# Set VERBOSE_LOGCAT=1 to debug every line of logcat it tries to parse.
+logcat_wait_for_pattern() {
+ local timeout="$1"
+ local timestamp="$2"
+ local pattern="$3"
+
+ local logcat_fd
+
+ coproc logcat_fd {
+ kill_children_quietly() {
+ kill "$logcat_pidd"
+ wait "$logcat_pidd" 2>/dev/null
+ }
+
+ trap 'kill_children_quietly' EXIT # kill logcat when this coproc is killed.
+
+ # run logcat in the background so it can be killed.
+ logcat_from_timestamp_bg "$timestamp"
+ logcat_pidd=$logcat_from_timestamp_pid
+ wait "$logcat_pidd"
+ }
+ local logcat_pid="$!"
+ verbose_print "[LOGCAT] Spawn pid $logcat_pid"
+
+ local timeout_ts="$(date -d "now + ${timeout} seconds" '+%s')"
+ local now_ts="0"
+
+ local return_code=1
+
+ verbose_print "logcat_wait_for_pattern begin"
+
+ while read -t "$timeout" -r -u "${logcat_fd[0]}" logcat_output; do
+ if (( $VERBOSE_LOGCAT )); then
+ verbose_print "LOGCAT: $logcat_output"
+ fi
+ if [[ "$logcat_output:" == *"$pattern"* ]]; then
+ verbose_print "LOGCAT: " "$logcat_output"
+ verbose_print "WE DID SEE PATTERN" '<<' "$pattern" '>>.'
+ return_code=0
+ break
+ fi
+ now_ts="$(date -d "now" '+%s')"
+ if (( now_ts >= timeout_ts )); then
+ verbose_print "DID TIMEOUT BEFORE SEEING ANYTHING (timeout=$timeout seconds) " '<<' "$pattern" '>>.'
+ break
+ fi
+ done
+
+ # Don't leave logcat lying around since it will keep going.
+ kill "$logcat_pid"
+ # Suppress annoying 'Terminated...' message.
+ wait "$logcat_pid" 2>/dev/null
+
+ verbose_print "[LOGCAT] $logcat_pid should be killed"
+
+ return $return_code
+}
diff --git a/startop/scripts/app_startup/run_app_with_prefetch b/startop/scripts/app_startup/run_app_with_prefetch
index 56bffa8..18aaf32 100755
--- a/startop/scripts/app_startup/run_app_with_prefetch
+++ b/startop/scripts/app_startup/run_app_with_prefetch
@@ -24,7 +24,7 @@
-v, --verbose enable extra verbose printing
-i, --input <file> trace file protobuf (default 'TraceFile.pb')
-r, --readahead <mode> cold, warm, fadvise, mlock (default 'warm')
- -w, --when <when> aot or jit (default 'aot')
+ -w, --when <when> aot or jit (default 'jit')
-c, --count <count> how many times to run (default 1)
-s, --sleep <sec> how long to sleep after readahead
-t, --timeout <sec> how many seconds to timeout in between each app run (default 10)
@@ -33,7 +33,7 @@
}
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-source "$DIR/lib/common"
+source "$DIR/../iorap/common"
needs_trace_file="n"
input_file=""
@@ -43,7 +43,7 @@
sleep_time=2
timeout=10
output="" # stdout by default
-when="aot"
+when="jit"
parse_arguments() {
while [[ $# -gt 0 ]]; do
case "$1" in
@@ -176,16 +176,19 @@
fi
verbose_print "Package was in path '$package_path'"
-keep_application_trace_file=n
application_trace_file_path="$package_path/TraceFile.pb"
trace_file_directory="$package_path"
if [[ $needs_trace_file == y ]]; then
# system server always passes down the package path in a hardcoded spot.
if [[ $when == "jit" ]]; then
- verbose_print adb push "$input_file" "$application_trace_file_path"
- adb push "$input_file" "$application_trace_file_path"
+ if ! iorapd_compiler_install_trace_file "$package" "$activity" "$input_file"; then
+ echo "Error: Failed to install compiled TraceFile.pb for '$package/$activity'" >&2
+ exit 1
+ fi
keep_application_trace_file="y"
else
+ echo "TODO: --readahead=aot is non-functional and needs to be fixed." >&2
+ exit 1
# otherwise use a temporary directory to get normal non-jit behavior.
trace_file_directory="/data/local/tmp/prefetch/$package"
adb shell mkdir -p "$trace_file_directory"
@@ -198,7 +201,7 @@
# otherwise system server activity hints will kick in
# and the new just-in-time app pre-warmup will happen.
if [[ $keep_application_trace_file == "n" ]]; then
- adb shell "[[ -f '$application_trace_file_path' ]] && rm '$application_trace_file_path'"
+ iorapd_compiler_purge_trace_file "$package" "$activity"
fi
# Perform AOT readahead/pinning/etc when an application is about to be launched.
@@ -242,12 +245,20 @@
fi
}
-perform_aot_cleanup() {
+# Perform cleanup at the end of each loop iteration.
+perform_post_launch_cleanup() {
local the_when="$1" # user: aot, jit
local the_mode="$2" # warm, cold, fadvise, mlock, etc.
+ local logcat_timestamp="$3" # timestamp from before am start.
if [[ $the_when != "aot" ]]; then
- # TODO: just in time implementation.. should probably use system server.
+ if [[ $the_mode != 'warm' && $the_mode != 'cold' ]]; then
+ # Validate that readahead completes.
+ # If this fails for some reason, then this will also discard the timing of the run.
+ iorapd_readahead_wait_until_finished "$package" "$activity" "$logcat_timestamp" "$timeout"
+ return $?
+ fi
+ # Don't need to do anything for warm or cold.
return 0
fi
@@ -321,6 +332,8 @@
verbose_print "Running with timeout $timeout"
+ pre_launch_timestamp="$(logcat_save_timestamp)"
+
# TODO: multiple metrics output.
total_time="$(timeout $timeout $DIR/launch_application "$package" "$activity")"
@@ -330,7 +343,13 @@
continue
fi
- perform_aot_cleanup "$when" "$mode"
+ perform_post_launch_cleanup "$when" "$mode" "$pre_launch_timestamp"
+
+ if [[ $? -ne 0 ]]; then
+ echo "WARNING: Skip bad cleanup, try iteration again." >&2
+ ((i=i-1))
+ continue
+ fi
echo "Iteration $i. Total time was: $total_time"
@@ -354,9 +373,9 @@
echo " Saved results to '$output'"
fi
-# Temporary hack around multiple activities being launched with different package paths (for same app):
-# Clean up all left-over TraceFile.pb
-adb shell 'for i in $(find /data/app -name TraceFile.pb); do rm \$i; done'
+if [[ $needs_trace_file == y ]] ; then
+ iorapd_compiler_purge_trace_file "$package" "$activity"
+fi
# Kill the process to ensure AM isn't keeping it around.
remote_pkill "$package"
diff --git a/startop/scripts/iorap/collector b/startop/scripts/iorap/collector
new file mode 100755
index 0000000..8d105ad
--- /dev/null
+++ b/startop/scripts/iorap/collector
@@ -0,0 +1,389 @@
+#!/bin/bash
+#
+# Copyright 2017, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+APP_STARTUP_DIR="$DIR/../app_startup/"
+source "$DIR/common"
+
+usage() {
+ cat <<EOF
+Usage: collector [OPTIONS]...
+
+Runs an application, causes an iorap trace to be collected for it, and then invokes the iorap
+compiler to generate a TraceFile.pb.
+
+ -p, --package package of the app to test
+ -a, --activity activity of the app to test
+ -h, --help usage information (this)
+ -v, --verbose enable extra verbose printing
+ -i, --inodes path to inodes file (system/extras/pagecache/pagecache.py -d inodes)
+ -b, --trace_buffer_size how big to make trace buffer size (default 32768)
+ -w, --wait_time how long to run systrace for (default 10) in seconds
+ -c, --compiler-filter override the compilation filter if set (default none)
+ -o, --output output trace file protobuf (default 'TraceFile.pb')
+EOF
+}
+
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+trace_buffer_size=32768
+wait_time=10
+comp_filter=""
+output_dest="TraceFile.pb"
+
+parse_arguments() {
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ -a|--activity)
+ activity="$2"
+ shift
+ ;;
+ -h|--help)
+ usage
+ exit 0
+ ;;
+ -p|--package)
+ package="$2"
+ shift
+ ;;
+ -i|--inodes)
+ inodes="$2"
+ shift
+ ;;
+ -b|--trace_buffer_size)
+ trace_buffer_size="$2"
+ shift
+ ;;
+ -w|--wait_time)
+ wait_time="$2"
+ shift
+ ;;
+ -c|--compiler-filter)
+ comp_filter="$2"
+ shift
+ ;;
+ -o|--output)
+ output_dest="$2"
+ shift
+ ;;
+ -v|--verbose)
+ verbose="y"
+ ;;
+ esac
+ shift
+ done
+}
+
+remote_pidof() {
+ local procname="$1"
+ adb shell ps | grep "$procname" | awk '{print $2;}'
+}
+
+remote_pkill() {
+ local procname="$1"
+ shift
+
+ local the_pids=$(remote_pidof "$procname")
+ local pid
+
+ for pid in $the_pids; do
+ verbose_print adb shell kill "$@" "$pid"
+ adb shell kill "$@" "$pid"
+ done
+}
+
+force_package_compilation() {
+ local arg_comp_filter="$1"
+ local arg_package="$2"
+
+ if [[ $arg_comp_filter == speed-profile ]]; then
+ # Force the running app to dump its profiles to disk.
+ remote_pkill "$arg_package" -SIGUSR1
+ sleep 1 # give some time for above to complete.
+ fi
+
+ adb shell cmd package compile -m "$arg_comp_filter" -f "$arg_package"
+}
+
+parse_package_dumpsys_line() {
+ local what_left="$1"
+ local what_right="$2"
+ local line="$3"
+
+ if [[ $line == *${what_left}*${what_right}* ]]; then
+ found="${line#*$what_left}"
+ found="${found%$what_right*}"
+ echo "$found"
+ return 0
+ fi
+
+ return 1
+}
+
+parse_package_dumpsys_section() {
+ local what_left="$1"
+ local what_right="$2"
+ shift
+ local lines="$@"
+
+ lines="${lines//$'\n'/}"
+
+ local new_lines=()
+
+ local current_line=""
+ local newline=n
+ local line
+ for line in "${lines[@]}"; do
+ if [[ $line == *: ]]; then
+ newline=y
+ current_line=""
+ new_lines+=("$current_line")
+
+ parse_package_dumpsys_line "$what_left" "$what_right" "$current_line" && return 0
+ else
+ # strip all spaces from the start
+ line="${line//$' '/}"
+ current_line+="$line"
+ #prepend to current line
+ fi
+ done
+ [[ "$current_line" != "" ]] && new_lines+=("$current_line")
+
+ parse_package_dumpsys_line "$what_left" "$what_right" "$current_line" && return 0
+
+ return 1
+}
+
+parse_package_compilation() {
+ local pkg="$1"
+# [com.google.android.apps.maps]
+
+ local compilation_filter
+ local is_prebuilt
+ local isa
+ local etc
+
+ local ret_code
+
+ read compilation_filter is_prebuilt isa etc <<< "$("$APP_STARTUP_DIR"/query_compiler_filter.py --package "$pkg")"
+ ret_code=$?
+
+ if [[ $ret_code -eq 0 && x$compilation_filter != x ]]; then
+ verbose_print "Package compilation info for $pkg was '$compilation_filter'"
+ echo "$compilation_filter"
+ return 0
+ else
+ verbose_print "query failed ret code $ret_code filter=$compilation_filter"
+ fi
+
+ return $ret_code
+}
+
+# Main entry point
+if [[ $# -eq 0 ]]; then
+ usage
+ exit 1
+else
+ parse_arguments "$@"
+
+ # if we do not have have package exit early with an error
+ [[ "$package" == "" ]] && echo "--package not specified" 1>&2 && exit 1
+
+ if [[ -z "$inodes" ]] || ! [[ -f $inodes ]]; then
+ echo "--inodes not specified" 1>&2
+ exit 1
+ fi
+
+ if [[ "$activity" == "" ]]; then
+ activity="$(get_activity_name "$package")"
+ if [[ "$activity" == "" ]]; then
+ echo "Activity name could not be found, invalid package name?" 1>&2
+ exit 1
+ else
+ verbose_print "Activity name inferred: " "$activity"
+ fi
+ fi
+fi
+
+adb root > /dev/null
+
+if [[ "$(adb shell getenforce)" != "Permissive" ]]; then
+ adb shell setenforce 0
+ adb shell stop
+ adb shell start
+ adb wait-for-device
+fi
+
+compilation_was="$(parse_package_compilation "$package")"
+if [[ $? -ne 0 ]]; then
+ echo "Could not determine package compilation filter; was this package installed?" >&2
+ exit 1
+fi
+verbose_print "Package compilation: $compilation_was"
+
+# Cannot downgrade (e.g. from speed-profile to quicken) without forceful recompilation.
+# Forceful recompilation will recompile even if compilation filter was unchanged.
+# Therefore avoid recompiling unless the filter is actually different than what we asked for.
+if [[ "x$comp_filter" != "x" ]] && [[ "$compilation_was" != "$comp_filter" ]]; then
+ echo "Current compilation filter is '$compilation_was'; force recompile to '$comp_filter'" >&2
+ #TODO: this matching seems hopelessly broken, it will always recompile.
+
+ force_package_compilation "$comp_filter" "$package"
+fi
+
+# Drop all caches prior to beginning a systrace, otherwise we won't record anything already in pagecache.
+adb shell "echo 3 > /proc/sys/vm/drop_caches"
+
+trace_tmp_file="$(mktemp -t trace.XXXXXXXXX.html)"
+
+function finish {
+ [[ -f "$trace_tmp_file" ]] && rm "$trace_tmp_file"
+}
+trap finish EXIT
+
+launch_application_and_wait_for_trace() {
+ local package="$1"
+ local activity="$2"
+ local timeout=30 # seconds
+
+ # Ensure application isn't running already.
+ remote_pkill "$package"
+
+ local time_now="$(logcat_save_timestamp)"
+ local retcode=0
+
+ verbose_print "now launching application"
+ # Launch an application
+ "$APP_STARTUP_DIR"/launch_application "$package" "$activity"
+ retcode=$?
+ if [[ $retcode -ne 0 ]]; then
+ echo "FATAL: Application launch failed." >&2
+ return $retcode
+ fi
+
+ # This blocks until 'am start' returns at which point the application is
+ # already to be considered "started" as the first frame has been drawn.
+
+ # TODO: check for cold start w.r.t to activitymanager?
+
+ # Wait for application to start from the point of view of ActivityTaskManager.
+ local pattern="ActivityTaskManager: Displayed $package"
+ logcat_wait_for_pattern "$timeout" "$time_now" "$pattern"
+ retcode=$?
+ if [[ $retcode -ne 0 ]]; then
+ echo "FATAL: Could not find '$pattern' in logcat." >&2
+ return $retcode
+ fi
+
+ # Wait for iorapd to finish writing out the perfetto traces for this app.
+ iorapd_perfetto_wait_for_app_trace "$package" "$activity" "$timeout" "$time_now"
+ retcode=$?
+ if [[ $retcode -ne 0 ]]; then
+ echo "FATAL: Could not save perfetto app trace file." >&2
+ return $retcode
+ fi
+
+ verbose_print "iorapd has finished collecting app trace file for $package/$activity"
+}
+
+collector_main() {
+ # don't even bother trying to run anything until the screen is unlocked.
+ "$APP_STARTUP_DIR"/unlock_screen
+
+ # Don't mutate state while iorapd is running.
+ iorapd_stop || return $?
+
+ # Remove all existing metadata for a package/activity in iorapd.
+ iorapd_perfetto_purge_app_trace "$package" "$activity" || return $?
+ iorapd_compiler_purge_trace_file "$package" "$activity" || return $?
+
+ iorapd_perfetto_enable || return $?
+ iorapd_start || return $?
+
+ # Wait for perfetto trace to finished writing itself out.
+ launch_application_and_wait_for_trace "$package" "$activity" || return $?
+
+ # Compile the trace so that the next app run can use prefetching.
+ iorapd_compiler_for_app_trace "$package" "$activity" "$inodes" || return $?
+
+ # Save TraceFile.pb to local file.
+ iorapd_compiler_pull_trace_file "$package" "$activity" "$output_dest" || return $?
+ # Remove the TraceFile.pb from the device.
+ iorapd_compiler_purge_trace_file "$package" "$activity" || return $?
+
+ # TODO: better transactional support for restoring iorapd global properties
+ iorapd_perfetto_disable || return $?
+}
+
+collector_main "$@"
+
+verbose_print "Collector finished. Children: "
+if [[ $verbose == y ]]; then
+ jobs -p
+ ps f -g$$
+fi
+
+exit $?
+
+
+verbose_print "About to begin systrace"
+coproc systrace_fd {
+ # Disable stdout buffering since we need to know the output of systrace RIGHT AWAY.
+ stdbuf -oL "$ANDROID_BUILD_TOP"/external/chromium-trace/systrace.py --target=android -b "$trace_buffer_size" -t "$wait_time" am pagecache dalvik -o "$trace_tmp_file"
+}
+
+verbose_print "Systrace began"
+
+systrace_pid="$!"
+
+while read -r -u "${systrace_fd[0]}" systrace_output; do
+ verbose_print "$systrace_output"
+ if [[ "$systrace_output" == *"Starting tracing"* ]]; then
+ verbose_print "WE DID SEE STARTING TRACING."
+ break
+ fi
+done
+# Systrace has begun recording the tracing.
+# Run the application and collect the results.
+
+am_output="$(adb shell am start -S -W "$package"/"$activity")"
+if [[ $? -ne 0 ]]; then
+ echo "am start failed" >&2
+
+ exit 1
+fi
+
+verbose_print "$am_output"
+total_time="$(echo "$am_output" | grep 'TotalTime:' | sed 's/TotalTime: //g')"
+verbose_print "total time: $total_time"
+
+# Now wait for systrace to finish.
+
+wait "$systrace_pid" || { echo "Systrace finished before am start was finished, try a longer --wait_time"; exit 1; }
+verbose_print "Systrace has now finished"
+verbose_print "$(ls -la "$trace_tmp_file")"
+
+
+iorapd_perfetto_disable
+
+# Now that systrace has finished, convert the trace file html file to a protobuf.
+
+"$ANDROID_BUILD_TOP"/system/iorap/src/py/collector/trace_parser.py -i "$inodes" -t "$trace_tmp_file" -o "$output_dest" || exit 1
+
+echo "Trace file collection complete, trace file saved to \"$output_dest\"!" >&2
+
+finish
diff --git a/startop/scripts/iorap/common b/startop/scripts/iorap/common
new file mode 100755
index 0000000..6dde3d5
--- /dev/null
+++ b/startop/scripts/iorap/common
@@ -0,0 +1,207 @@
+#!/bin/bash
+#
+# Copyright 2019, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DIR_IORAP_COMMON="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+APP_STARTUP_DIR="$DIR/../app_startup/"
+source "$APP_STARTUP_DIR/lib/common"
+
+IORAPD_DATA_PATH="/data/misc/iorapd"
+
+iorapd_start() {
+ verbose_print 'iorapd_start'
+ adb shell start iorapd
+ sleep 1
+ # TODO: block until logcat prints successfully connecting
+}
+
+iorapd_stop() {
+ verbose_print 'iorapd_stop'
+ adb shell stop iorapd
+}
+
+iorapd_reset() {
+ iorapd_stop
+ iorapd_start
+}
+
+# Enable perfetto tracing.
+# Subsequent launches of an application will record a perfetto trace protobuf.
+iorapd_perfetto_enable() {
+ verbose_print 'enable perfetto'
+ adb shell setprop iorapd.perfetto.enable true
+ iorapd_reset # iorapd only reads this flag when initializing
+}
+
+# Enable perfetto tracing.
+# Subsequent launches of applications will no longer record perfetto trace protobufs.
+iorapd_perfetto_disable() {
+ verbose_print 'disable perfetto'
+ adb shell setprop iorapd.perfetto.enable false
+ iorapd_reset # iorapd only reads this flag when initializing
+}
+
+_iorapd_path_to_data_file() {
+ local package="$1"
+ local activity="$2"
+ local suffix="$3"
+
+ # Match logic of 'AppComponentName' in iorap::compiler C++ code.
+ echo "${IORAPD_DATA_PATH}/${package}%2F${activity}.${suffix}"
+}
+
+iorapd_perfetto_wait_for_app_trace() {
+ local package="$1"
+ local activity="$2"
+ local timeout="$3"
+ local timestamp="$4"
+
+ local remote_path="$(_iorapd_path_to_data_file "$package" "$activity" "perfetto_trace.pb")"
+
+ verbose_print "iorapd_perfetto_wait_for_app_trace on file '$remote_path'"
+
+ # see event_manager.cc
+ local pattern="Perfetto TraceBuffer saved to file: $remote_path"
+ logcat_wait_for_pattern "$timeout" "$timestamp" "$pattern"
+}
+
+# Purge all perfetto traces for a given application.
+iorapd_perfetto_purge_app_trace() {
+ local package="$1"
+ local activity="$2"
+
+ local remote_path="$(_iorapd_path_to_data_file "$package" "$activity" "perfetto_trace.pb")"
+
+ verbose_print 'iorapd-perfetto: purge app trace in ' "$remote_path"
+ adb shell "[[ -f '$remote_path' ]] && rm -f '$remote_path' || exit 0"
+}
+
+# Compile a perfetto trace for a given application.
+# This requires the app has run at least once with perfetto tracing enabled.
+iorapd_compiler_for_app_trace() {
+ local package="$1"
+ local activity="$2"
+ local inodes="$3" # local path
+
+ # remote path calculations
+ local input_path="$(_iorapd_path_to_data_file "$package" "$activity" "perfetto_trace.pb")"
+ local compiled_path="$(_iorapd_path_to_data_file "$package" "$activity" "compiled_trace.tmp.pb")"
+ local compiled_path_final="$(_iorapd_path_to_data_file "$package" "$activity" "compiled_trace.pb")"
+
+ if ! adb shell "[[ -f '$input_path' ]]"; then
+ echo "Error: Missing perfetto traces; nothing to compile. Expected: '$input_path'" >&2
+ return 1
+ fi
+
+ if ! [[ -f $inodes ]]; then
+ # We could compile using 'diskscan' but it's non-deterministic, so refuse instead.
+ echo "Error: Missing inodes textcache at '$inodes'; refusing to compile." >&2
+ return 1
+ fi
+
+ # inodes file needs to be on the device for iorap.cmd.compiler to access it
+ local remote_inodes=/data/local/tmp/prefetch/inodes.txt
+ adb shell "mkdir -p \"$(dirname "$remote_inodes")\"" || return 1
+ adb push "$inodes" "$remote_inodes"
+
+ verbose_print 'iorapd-compiler: compile app trace in ' "$input_path"
+ verbose_print adb shell "iorap.cmd.compiler '$input_path' --inode-textcache '$remote_inodes' --output-proto '$compiled_path'"
+ adb shell "iorap.cmd.compiler '$input_path' --inode-textcache '$remote_inodes' --output-proto '$compiled_path'"
+ retcode=$?
+
+ # Don't overwrite the true 'compiled_trace.pb' unless the compiler completed without error.
+ # TODO: The native compiler code should be handling its own transaction-safety.
+ if [[ $retcode -eq 0 ]]; then
+ adb shell "mv '$compiled_path' '$compiled_path_final'"
+ else
+ adb shell "[[ -f '$compiled_path' ]] && rm -f '$compiled_path'"
+ fi
+
+ # Clean up inodes file we just pushed.
+ adb shell "[[ -f '$remote_inodes' ]] && rm -f '$remote_inodes'"
+
+ return $retcode
+}
+
+# Pull the remote compiled trace file into a local file.
+iorapd_compiler_pull_trace_file() {
+ local package="$1"
+ local activity="$2"
+ local output_file="$3" # local path
+
+ local compiled_path="$(_iorapd_path_to_data_file "$package" "$activity" "compiled_trace.pb")"
+
+ if ! adb shell "[[ -f '$compiled_path' ]]"; then
+ echo "Error: Remote path '$compiled_path' invalid" >&2
+ return 1
+ fi
+ if ! mkdir -p "$(dirname "$output_file")"; then
+ echo "Error: Fail to make output directory for '$output_file'" >&2
+ return 1
+ fi
+ verbose_print adb pull "$compiled_path" "$output_file"
+ adb pull "$compiled_path" "$output_file"
+}
+
+# Install a compiled trace file.
+iorapd_compiler_install_trace_file() {
+ local package="$1"
+ local activity="$2"
+ local input_file="$3" # local path
+
+ # remote path calculations
+ local compiled_path="$(_iorapd_path_to_data_file "$package" "$activity" "compiled_trace.pb")"
+
+ if ! [[ -f $input_file ]]; then
+ echo "Error: File '$input_file' does not exist." >&2
+ return 1
+ fi
+
+ adb shell "mkdir -p \"$(dirname "$compiled_path")\"" || return 1
+
+ verbose_print adb push "$input_file" "$compiled_path"
+ adb push "$input_file" "$compiled_path"
+}
+
+iorapd_compiler_purge_trace_file() {
+ local package="$1"
+ local activity="$2"
+ local input_file="$3" # local path
+
+ local remote_path="$(_iorapd_path_to_data_file "$package" "$activity" "compiled_trace.pb")"
+
+ adb shell "[[ -f '$remote_path' ]] && rm -f '$remote_path' || exit 0"
+}
+
+# Blocks until the readahead for the requested package/activity has finished.
+# This assumes that the trace file was already installed, and also that
+# the application launched but not completed yet.
+iorapd_readahead_wait_until_finished() {
+ local package="$1"
+ local activity="$2"
+ local timestamp="$3"
+ local timeout="$4"
+
+ if [[ $# -lt 4 ]]; then
+ echo "FATAL: Expected 4 arguments (actual $# $@)" >&2
+ exit 1
+ fi
+
+ local remote_path="$(_iorapd_path_to_data_file "$package" "$activity" "compiled_trace.pb")"
+
+ # See 'read_ahead.cc' LOG(INFO).
+ local pattern="ReadAhead completed ($remote_path)"
+ logcat_wait_for_pattern "$timeout" "$timestamp" "$pattern"
+}
diff --git a/startop/scripts/iorap/pull_textcache b/startop/scripts/iorap/pull_textcache
new file mode 100755
index 0000000..0554426
--- /dev/null
+++ b/startop/scripts/iorap/pull_textcache
@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Copyright 2019, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ $# -lt 1 ]]; then
+ echo "Usage: $0 <output-filename>" >&2
+ exit 1
+fi
+
+# see compiler/main.cc for list of roots
+adb shell iorap.inode2filename --output-format=textcache --output=/data/local/tmp/dumpcache --all --root=/system --root=/apex --root=/vendor --root=/data --root=/product --root=/metadata
+adb pull /data/local/tmp/dumpcache "$1"