Update boot image profile generation scripts
The old script is not longer suitable. It is replace by two scripts:
- one that can configure the device to capture boot image profiles
- and the second, which can process the raw boot image profiles
obtained after running CUJs and transform it to the expected build
system format.
Note that boot-image-profile-generate.sh should be invoked with
arguments based on the ammount of executed CUJs and the shape of
profile data. Some defaults are provided but these might not be
the best in all cases.
Test: run the scripts and check the data
Bug: 152574358
Change-Id: Ic814f22591d8861e7224f8deadbcc21427f94333
diff --git a/tools/boot-image-profile-configure-device.sh b/tools/boot-image-profile-configure-device.sh
new file mode 100755
index 0000000..081f442
--- /dev/null
+++ b/tools/boot-image-profile-configure-device.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This script configures a device for boot image profile
+#
+
+if [[ -z "$ANDROID_BUILD_TOP" ]]; then
+ echo "You must run on this after running envsetup.sh and launch target"
+ exit 1
+fi
+
+if [[ "$#" -lt 1 ]]; then
+ echo "Usage $0 <output-for-boot-zip>"
+ echo "Example: $0 boot.zip"
+ exit 1
+fi
+
+OUT_BOOT_ZIP="$1"
+
+echo "Changing dirs to the build top"
+cd "$ANDROID_BUILD_TOP"
+
+# Make dist in order to easily get the boot and system server dex files
+# This will be stored in $ANDROID_PRODUCT_OUT/boot.zip
+echo "Make dist"
+m dist
+echo "Copy boot.zip to $OUT_BOOT_ZIP"
+cp "$ANDROID_PRODUCT_OUT"/boot.zip $OUT_BOOT_ZIP
+
+echo "Setting properties and clearing existing profiles"
+# If the device needs to be rebooted, it is better to set the properties
+# via a local.prop file:
+# 1) create a local.prop file with the content
+# dalvik.vm.profilebootclasspath=true
+# dalvik.vm.profilesystemserver=true
+# 2) adb push local.prop /data/
+# adb shell chmod 0750 /data/local.prop
+# adb reboot
+
+adb root
+adb shell stop
+adb shell setprop dalvik.vm.profilebootclasspath true
+adb shell setprop dalvik.vm.profilesystemserver true
+adb shell find "/data/misc/profiles -name *.prof -exec truncate -s 0 {} \;"
+adb shell start
\ No newline at end of file
diff --git a/tools/boot-image-profile-extract-profile.sh b/tools/boot-image-profile-extract-profile.sh
new file mode 100755
index 0000000..e050e36
--- /dev/null
+++ b/tools/boot-image-profile-extract-profile.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This script extracts the boot image profile from a previously configured device,
+# which executed the critical user journyes.
+#
+
+if [[ "$#" -lt 1 ]]; then
+ echo "Usage $0 <output-profile>"
+ echo "Example: $0 android.prof"
+ exit 1
+fi
+
+OUT_PROFILE="$1"
+
+echo "Snapshoting platform profiles"
+adb shell cmd package snapshot-profile android
+adb pull /data/misc/profman/android.prof "$OUT_PROFILE"
\ No newline at end of file
diff --git a/tools/boot-image-profile-generate.sh b/tools/boot-image-profile-generate.sh
new file mode 100755
index 0000000..e21048e
--- /dev/null
+++ b/tools/boot-image-profile-generate.sh
@@ -0,0 +1,152 @@
+#!/bin/bash
+#
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This script creates the final boot image profile (suitable to include in the platform build).
+# The input to the script are:
+# 1) the boot.zip file which contains the boot classpath and system server jars.
+# This file can be obtained from running `m dist` or by configuring the device with
+# the `art/tools/boot-image-profile-configure-device.sh` script.
+# 2) the preloaded classes blacklist which specify what clases should not be preloaded
+# in Zygote. Usually located in usually in frameworks/base/config/preloaded-classes-blacklist
+# 3) a list of raw boot image profiles extracted from devices. An example how to do that is
+# by running `art/tools/boot-image-profile-extract-profile.sh` script.
+#
+# It is strongly recommended that you make use of extensive critical user journeys flows in order
+# to capture the raw boot image profiles described in #3.
+#
+# NOTE: by default, the script uses default arguments for producing the boot image profiles.
+# You might want to adjust the default generation arguments based on the shape of profile
+# and based on the metrics that matter for each product.
+#
+
+if [[ -z "$ANDROID_BUILD_TOP" ]]; then
+ echo "You must run on this after running envsetup.sh and launch target"
+ exit 1
+fi
+
+if [[ "$#" -lt 4 ]]; then
+ echo "Usage $0 <output-dir> <boot.zip-location> <preloaded-blacklist-location> <profile-input1> <profile-input2> ... <profman args>"
+ echo "Without any profman args the script will use defaults."
+ echo "Example: $0 output-dir boot.zip frameworks/base/config/preloaded-classes-blacklist android1.prof android2.prof"
+ echo " $0 output-dir boot.zip frameworks/base/config/preloaded-classes-blacklist android.prof --profman-arg --upgrade-startup-to-hot=true"
+ echo "preloaded.black-list is usually in frameworks/base/config/preloaded-classes-blacklist"
+ exit 1
+fi
+
+echo "Creating work dir"
+WORK_DIR=/tmp/android-bcp
+mkdir -p "$WORK_DIR"
+
+OUT_DIR="$1"
+BOOT_ZIP="$2"
+PRELOADED_BLACKLIST="$3"
+shift 3
+
+IN
+
+# Read the profile input args.
+profman_profile_input_args=()
+while [[ "$#" -ge 1 ]] && [[ ! "$1" = '--profman-arg' ]]; do
+ profman_profile_input_args+=("--profile-file=$1")
+ shift
+done
+
+# Read the profman args.
+profman_args=()
+while [[ "$#" -ge 2 ]] && [[ "$1" = '--profman-arg' ]]; do
+ profman_args+=("$2")
+ shift 2
+done
+
+OUT_BOOT_PROFILE="$OUT_DIR"/boot-image-profile.txt
+OUT_PRELOADED_CLASSES="$OUT_DIR"/preloaded-classes
+OUT_SYSTEM_SERVER="$OUT_DIR"/art-profile
+
+echo "Changing dirs to the build top"
+cd "$ANDROID_BUILD_TOP"
+
+echo "Unziping boot.zip"
+BOOT_UNZIP_DIR="$WORK_DIR"/boot-dex
+ART_JARS="$BOOT_UNZIP_DIR"/dex_artjars_input
+BOOT_JARS="$BOOT_UNZIP_DIR"/dex_bootjars_input
+SYSTEM_SERVER_JAR="$BOOT_UNZIP_DIR"/system/framework/services.jar
+
+unzip -o "$BOOT_ZIP" -d "$BOOT_UNZIP_DIR"
+
+echo "Processing boot image jar files"
+jar_args=()
+for entry in "$ART_JARS"/*
+do
+ jar_args+=("--apk=$entry")
+done
+for entry in "$BOOT_JARS"/*
+do
+ jar_args+=("--apk=$entry")
+done
+profman_args+=("${jar_args[@]}")
+
+echo "Running profman for boot image profiles"
+# NOTE:
+# You might want to adjust the default generation arguments based on the data
+# For example, to update the selection thresholds you could specify:
+# --method-threshold=10 \
+# --class-threshold=10 \
+# --preloaded-class-threshold=10 \
+# --special-package=android:1 \
+# --special-package=com.android.systemui:1 \
+# The threshold is percentage of total aggregation, that is, a method/class is
+# included in the profile only if it's used by at least x% of the packages.
+# (from 0% - include everything to 100% - include only the items that
+# are used by all packages on device).
+# The --special-package allows you to give a prioriority to certain packages,
+# meaning, if the methods is used by that package then the algorithm will use a
+# different selection thresholds.
+# (system server is identified as the "android" package)
+profman \
+ --generate-boot-image-profile \
+ "${profman_profile_input_args[@]}" \
+ --out-profile-path="$OUT_BOOT_PROFILE" \
+ --out-preloaded-classes-path="$OUT_PRELOADED_CLASSES" \
+ --preloaded-classes-blacklist="$PRELOADED_BLACKLIST" \
+ --special-package=android:1 \
+ --special-package=com.android.systemui:1 \
+ "${profman_args[@]}"
+
+echo "Done boot image profile"
+
+echo "Running profman for system server"
+# For system server profile we want to include everything usually
+# We also don't have a preloaded-classes file for it, so we ignore the argument.
+profman \
+ --generate-boot-image-profile \
+ "${profman_profile_input_args[@]}" \
+ --out-profile-path="$OUT_SYSTEM_SERVER" \
+ --apk="$SYSTEM_SERVER_JAR" \
+ --method-threshold=0 \
+ --class-threshold=0
+
+echo "Done system server"
+
+echo ""
+echo "Boot profile methods+classes count: $(wc -l $OUT_BOOT_PROFILE)"
+echo "Preloaded classes count: $(wc -l $OUT_PRELOADED_CLASSES)"
+echo "System server profile methods+classes count: $(wc -l $OUT_SYSTEM_SERVER)"
+
+CLEAN_UP="${CLEAN_UP:-true}"
+if [[ "$CLEAN_UP" = "true" ]]; then
+ rm -rf "$WORK_DIR"
+fi
\ No newline at end of file
diff --git a/tools/generate-boot-image-profile.sh b/tools/generate-boot-image-profile.sh
deleted file mode 100755
index 0c4f29b..0000000
--- a/tools/generate-boot-image-profile.sh
+++ /dev/null
@@ -1,103 +0,0 @@
-#!/bin/bash
-#
-# Copyright (C) 2017 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# This script creates a boot image profile based on input profiles.
-#
-
-if [[ "$#" -lt 2 ]]; then
- echo "Usage $0 <output> <profman args> <profiles>+"
- echo "Also outputs <output>.txt and <output>.preloaded-classes"
- echo 'Example: generate-boot-image-profile.sh boot.prof --profman-arg --boot-image-sampled-method-threshold=1 profiles/cur/0/*/primary.prof'
- exit 1
-fi
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-TOP="$DIR/../.."
-source "${TOP}/build/envsetup.sh" >&/dev/null # import get_build_var
-
-OUT_PROFILE=$1
-shift
-
-# Read the profman args.
-profman_args=()
-while [[ "$#" -ge 2 ]] && [[ "$1" = '--profman-arg' ]]; do
- profman_args+=("$2")
- shift 2
-done
-
-# Remaining args are all the profiles.
-for file in "$@"; do
- if [[ -s $file ]]; then
- profman_args+=("--profile-file=$file")
- fi
-done
-
-# b/139391334: the stem of framework-minus-apex is framework
-real_jar_name() {
- if [[ "$1" == "framework-minus-apex" ]]; then
- echo "framework"
- else
- echo "$1"
- fi
-}
-
-# Boot jars have hidden API access flags which do not pass dex file
-# verification. Skip it.
-jar_args=()
-boot_jars=$("$ANDROID_BUILD_TOP"/art/tools/bootjars.sh --target)
-product_out=$ANDROID_BUILD_TOP/$(get_build_var PRODUCT_OUT)
-for pair in $boot_jars; do
- words=( $(echo $pair | tr ':' ' ') )
- if [[ ${#words[@]} -eq 2 ]]; then
- # format in Android > R: <apex>:<jar>
- apex="${words[0]}"
- name="$(real_jar_name ${words[1]})"
- case "$apex" in
- platform*) subdir=system/framework ;;
- system_ext*) subdir=system_ext/framework ;;
- *) subdir=apex/$apex/javalib ;;
- esac
- filename="$product_out/$subdir/$name.jar"
- else
- # format in Android <= R: <jar>, have to infer location with `find`
- name="$(real_jar_name ${words[0]})"
- filename="$(find $product_out -name $name.jar 2>/dev/null)"
- if [[ $(echo "$filename" | wc -w ) -ne 1 ]]; then
- echo "expected to find $name.jar, got '$filename'"
- exit 1
- fi
- fi
- jar_args+=("--apk=$filename")
- jar_args+=("--dex-location=$filename")
-done
-profman_args+=("${jar_args[@]}")
-
-# Generate the profile.
-"$ANDROID_HOST_OUT/bin/profman" --generate-boot-image-profile "--reference-profile-file=$OUT_PROFILE" "${profman_args[@]}"
-
-# Convert it to text.
-echo Dumping profile to $OUT_PROFILE.txt
-"$ANDROID_HOST_OUT/bin/profman" --dump-classes-and-methods "--profile-file=$OUT_PROFILE" "${jar_args[@]}" > "$OUT_PROFILE.txt"
-
-# Generate preloaded classes
-# Filter only classes by using grep -v
-# Remove first and last characters L and ;
-# Replace / with . to make dot format
-grep -v "\\->" "$OUT_PROFILE.txt" | sed 's/.\(.*\)./\1/g' | tr "/" "." > "$OUT_PROFILE.preloaded-classes"
-
-# You may need to filter some classes out since creating threads is not allowed in the zygote.
-# i.e. using: grep -v -E '(android.net.ConnectivityThread\$Singleton)'