Merge "Add logging to app startup to find flaky tests."
diff --git a/core/java/android/content/res/Configuration.java b/core/java/android/content/res/Configuration.java
index ac1cbd4..9cf54f4 100644
--- a/core/java/android/content/res/Configuration.java
+++ b/core/java/android/content/res/Configuration.java
@@ -23,7 +23,6 @@
import static android.content.ConfigurationProto.KEYBOARD;
import static android.content.ConfigurationProto.KEYBOARD_HIDDEN;
import static android.content.ConfigurationProto.LOCALES;
-import static android.content.ConfigurationProto.LOCALE_LIST;
import static android.content.ConfigurationProto.MCC;
import static android.content.ConfigurationProto.MNC;
import static android.content.ConfigurationProto.NAVIGATION;
@@ -1112,7 +1111,7 @@
protoOutputStream.write(MCC, mcc);
protoOutputStream.write(MNC, mnc);
if (mLocaleList != null) {
- protoOutputStream.write(LOCALE_LIST, mLocaleList.toLanguageTags());
+ mLocaleList.writeToProto(protoOutputStream, LOCALES);
}
protoOutputStream.write(SCREEN_LAYOUT, screenLayout);
protoOutputStream.write(COLOR_MODE, colorMode);
@@ -1284,14 +1283,6 @@
case (int) WINDOW_CONFIGURATION:
windowConfiguration.readFromProto(protoInputStream, WINDOW_CONFIGURATION);
break;
- case (int) LOCALE_LIST:
- try {
- setLocales(LocaleList.forLanguageTags(protoInputStream.readString(
- LOCALE_LIST)));
- } catch (Exception e) {
- Slog.e(TAG, "error parsing locale list in configuration.", e);
- }
- break;
}
}
} finally {
diff --git a/core/java/android/os/LocaleList.java b/core/java/android/os/LocaleList.java
index 0de09ef..7782753 100644
--- a/core/java/android/os/LocaleList.java
+++ b/core/java/android/os/LocaleList.java
@@ -21,7 +21,9 @@
import android.annotation.Nullable;
import android.annotation.Size;
import android.annotation.UnsupportedAppUsage;
+import android.content.LocaleProto;
import android.icu.util.ULocale;
+import android.util.proto.ProtoOutputStream;
import com.android.internal.annotations.GuardedBy;
@@ -141,6 +143,26 @@
}
/**
+ * Helper to write LocaleList to a protocol buffer output stream. Assumes the parent
+ * protobuf has declared the locale as repeated.
+ *
+ * @param protoOutputStream Stream to write the locale to.
+ * @param fieldId Field Id of the Locale as defined in the parent message.
+ * @hide
+ */
+ public void writeToProto(ProtoOutputStream protoOutputStream, long fieldId) {
+ for (int i = 0; i < mList.length; i++) {
+ final Locale locale = mList[i];
+ final long token = protoOutputStream.start(fieldId);
+ protoOutputStream.write(LocaleProto.LANGUAGE, locale.getLanguage());
+ protoOutputStream.write(LocaleProto.COUNTRY, locale.getCountry());
+ protoOutputStream.write(LocaleProto.VARIANT, locale.getVariant());
+ protoOutputStream.write(LocaleProto.SCRIPT, locale.getScript());
+ protoOutputStream.end(token);
+ }
+ }
+
+ /**
* Retrieves a String representation of the language tags in this list.
*/
@NonNull
diff --git a/core/proto/android/content/configuration.proto b/core/proto/android/content/configuration.proto
index 7fa0ff6..57ced09 100644
--- a/core/proto/android/content/configuration.proto
+++ b/core/proto/android/content/configuration.proto
@@ -32,7 +32,7 @@
optional float font_scale = 1;
optional uint32 mcc = 2;
optional uint32 mnc = 3 [ (.android.privacy).dest = DEST_EXPLICIT ];
- repeated LocaleProto locales = 4 [deprecated = true];
+ repeated LocaleProto locales = 4;
optional uint32 screen_layout = 5;
optional uint32 color_mode = 6;
optional uint32 touchscreen = 7;
@@ -48,7 +48,6 @@
optional uint32 smallest_screen_width_dp = 17;
optional uint32 density_dpi = 18;
optional .android.app.WindowConfigurationProto window_configuration = 19;
- optional string locale_list = 20;
}
/**
diff --git a/core/proto/android/content/locale.proto b/core/proto/android/content/locale.proto
index a8f2a13..bae6ec1 100644
--- a/core/proto/android/content/locale.proto
+++ b/core/proto/android/content/locale.proto
@@ -22,7 +22,6 @@
package android.content;
message LocaleProto {
- option deprecated = true;
option (.android.msg_privacy).dest = DEST_AUTOMATIC;
optional string language = 1;
diff --git a/core/tests/coretests/src/android/content/res/ConfigurationTest.java b/core/tests/coretests/src/android/content/res/ConfigurationTest.java
index 4a93f42..2fc3e36 100644
--- a/core/tests/coretests/src/android/content/res/ConfigurationTest.java
+++ b/core/tests/coretests/src/android/content/res/ConfigurationTest.java
@@ -16,29 +16,16 @@
package android.content.res;
-import android.content.Context;
-import android.os.LocaleList;
import android.platform.test.annotations.Presubmit;
-import android.util.AtomicFile;
-import android.util.proto.ProtoInputStream;
-import android.util.proto.ProtoOutputStream;
-import androidx.test.InstrumentationRegistry;
import androidx.test.filters.SmallTest;
-import com.android.server.usage.IntervalStatsProto;
-
import junit.framework.TestCase;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.util.Locale;
-
/**
* Build/install/run: bit FrameworksCoreTests:android.content.res.ConfigurationTest
*/
@@ -67,70 +54,4 @@
config2.updateFrom(config);
assertEquals(config2.screenLayout, Configuration.SCREENLAYOUT_COMPAT_NEEDED);
}
-
- @Test
- public void testReadWriteProto() throws Exception {
- final Context context = InstrumentationRegistry.getTargetContext();
- final File testDir = new File(context.getFilesDir(), "ConfigurationTest");
- testDir.mkdirs();
- final File proto = new File(testDir, "configs");
- if (proto.exists()) {
- proto.delete();
- }
-
- final Locale arabic = new Locale.Builder().setLocale(new Locale("ar", "AE")).build();
- final Locale urdu = new Locale.Builder().setLocale(new Locale("ur", "IN")).build();
- final Locale urduExtension = new Locale.Builder().setLocale(new Locale("ur", "IN"))
- .setExtension('u', "nu-latn").build();
- Configuration write = new Configuration();
- write.setLocales(new LocaleList(arabic, urdu, urduExtension));
- writeToProto(proto, write);
- assertTrue("Failed to write configs to proto.", proto.exists());
-
- final Configuration read = new Configuration();
- try {
- readFromProto(proto, read);
- } finally {
- proto.delete();
- }
-
- assertEquals("Missing locales in proto file written to disk.",
- read.getLocales().size(), write.getLocales().size());
- assertTrue("Arabic locale not found in Configuration locale list.",
- read.getLocales().indexOf(arabic) != -1);
- assertTrue("Urdu locale not found in Configuration locale list.",
- read.getLocales().indexOf(urdu) != -1);
- assertTrue("Urdu locale with extensions not found in Configuration locale list.",
- read.getLocales().indexOf(urduExtension) != -1);
- }
-
- private void writeToProto(File f, Configuration config) throws Exception {
- final AtomicFile af = new AtomicFile(f);
- FileOutputStream fos = af.startWrite();
- try {
- final ProtoOutputStream protoOut = new ProtoOutputStream(fos);
- final long token = protoOut.start(IntervalStatsProto.CONFIGURATIONS);
- config.writeToProto(protoOut, IntervalStatsProto.Configuration.CONFIG, false, false);
- protoOut.end(token);
- protoOut.flush();
- af.finishWrite(fos);
- fos = null;
- } finally {
- af.failWrite(fos);
- }
- }
-
- private void readFromProto(File f, Configuration config) throws Exception {
- final AtomicFile afRead = new AtomicFile(f);
- try (FileInputStream in = afRead.openRead()) {
- final ProtoInputStream protoIn = new ProtoInputStream(in);
- if (protoIn.nextField(IntervalStatsProto.CONFIGURATIONS)) {
- final long token = protoIn.start(IntervalStatsProto.CONFIGURATIONS);
- if (protoIn.nextField(IntervalStatsProto.Configuration.CONFIG)) {
- config.readFromProto(protoIn, IntervalStatsProto.Configuration.CONFIG);
- protoIn.end(token);
- }
- }
- }
- }
}
diff --git a/packages/PackageInstaller/src/com/android/packageinstaller/PackageInstallerActivity.java b/packages/PackageInstaller/src/com/android/packageinstaller/PackageInstallerActivity.java
index 55ff591..a2fa461 100644
--- a/packages/PackageInstaller/src/com/android/packageinstaller/PackageInstallerActivity.java
+++ b/packages/PackageInstaller/src/com/android/packageinstaller/PackageInstallerActivity.java
@@ -135,6 +135,7 @@
mEnableOk = true;
mOk.setEnabled(true);
+ mOk.setFilterTouchesWhenObscured(true);
}
/**
diff --git a/packages/SystemUI/src/com/android/keyguard/KeyguardUpdateMonitor.java b/packages/SystemUI/src/com/android/keyguard/KeyguardUpdateMonitor.java
index b964edf..720074b 100644
--- a/packages/SystemUI/src/com/android/keyguard/KeyguardUpdateMonitor.java
+++ b/packages/SystemUI/src/com/android/keyguard/KeyguardUpdateMonitor.java
@@ -217,8 +217,6 @@
}
}
- private static KeyguardUpdateMonitor sInstance;
-
private final Context mContext;
private final boolean mIsPrimaryUser;
HashMap<Integer, SimData> mSimDatas = new HashMap<Integer, SimData>();
@@ -1389,15 +1387,6 @@
Trace.endSection();
}
-
- /** Provides access to the static instance. */
- public static KeyguardUpdateMonitor getInstance(Context context) {
- if (sInstance == null) {
- sInstance = new KeyguardUpdateMonitor(context, Looper.getMainLooper());
- }
- return sInstance;
- }
-
protected void handleStartedGoingToSleep(int arg1) {
checkIsHandlerThread();
mLockIconPressed = false;
diff --git a/startop/scripts/app_startup/app_startup_runner.py b/startop/scripts/app_startup/app_startup_runner.py
index eb582f9..fa1c4e6 100755
--- a/startop/scripts/app_startup/app_startup_runner.py
+++ b/startop/scripts/app_startup/app_startup_runner.py
@@ -41,11 +41,12 @@
sys.path.append(os.path.dirname(DIR))
import lib.cmd_utils as cmd_utils
import lib.print_utils as print_utils
-import iorap.compiler as compiler
from app_startup.run_app_with_prefetch import PrefetchAppRunner
import app_startup.lib.args_utils as args_utils
from app_startup.lib.data_frame import DataFrame
from app_startup.lib.perfetto_trace_collector import PerfettoTraceCollector
+from iorap.compiler import CompilerType
+import iorap.compiler as compiler
# The following command line options participate in the combinatorial generation.
# All other arguments have a global effect.
@@ -58,8 +59,6 @@
CollectorPackageInfo = NamedTuple('CollectorPackageInfo',
[('package', str), ('compiler_filter', str)])
-_COMPILER_SCRIPT = os.path.join(os.path.dirname(os.path.dirname(
- os.path.realpath(__file__))), 'iorap/compiler.py')
# by 2; systrace starts up slowly.
_UNLOCK_SCREEN_SCRIPT = os.path.join(
@@ -135,6 +134,10 @@
action='append',
help='The trace duration (milliseconds) in '
'compilation')
+ optional_named.add_argument('--compiler-type', dest='compiler_type',
+ type=CompilerType, choices=list(CompilerType),
+ default=CompilerType.DEVICE,
+ help='The type of compiler.')
return parser.parse_args(argv)
@@ -211,26 +214,26 @@
return DataFrame(d)
-def compile_perfetto_trace(inodes_path: str,
+def build_ri_compiler_argv(inodes_path: str,
perfetto_trace_file: str,
- trace_duration: Optional[timedelta]) -> TextIO:
- compiler_trace_file = tempfile.NamedTemporaryFile()
- argv = [_COMPILER_SCRIPT, '-i', inodes_path, '--perfetto-trace',
- perfetto_trace_file, '-o', compiler_trace_file.name]
+ trace_duration: Optional[timedelta]
+ ) -> str:
+ argv = ['-i', inodes_path, '--perfetto-trace',
+ perfetto_trace_file]
if trace_duration is not None:
argv += ['--duration', str(int(trace_duration.total_seconds()
- * PerfettoTraceCollector.MS_PER_SEC))]
+ * PerfettoTraceCollector.MS_PER_SEC))]
print_utils.debug_print(argv)
- compiler.main(argv)
- return compiler_trace_file
+ return argv
def execute_run_using_perfetto_trace(collector_info,
run_combos: Iterable[RunCommandArgs],
simulate: bool,
inodes_path: str,
- timeout: int) -> DataFrame:
+ timeout: int,
+ compiler_type: CompilerType) -> DataFrame:
""" Executes run based on perfetto trace. """
passed, perfetto_trace_file = run_perfetto_collector(collector_info,
timeout,
@@ -244,9 +247,15 @@
if simulate:
compiler_trace_file = tempfile.NamedTemporaryFile()
else:
- compiler_trace_file = compile_perfetto_trace(inodes_path,
- perfetto_trace_file.name,
- combos.trace_duration)
+ ri_compiler_argv = build_ri_compiler_argv(inodes_path,
+ perfetto_trace_file.name,
+ combos.trace_duration)
+ compiler_trace_file = compiler.compile(compiler_type,
+ inodes_path,
+ ri_compiler_argv,
+ combos.package,
+ combos.activity)
+
with compiler_trace_file:
combos = combos._replace(input=compiler_trace_file.name)
print_utils.debug_print(combos)
@@ -261,7 +270,8 @@
grouped_run_combos: Iterable[Tuple[CollectorPackageInfo, Iterable[RunCommandArgs]]],
simulate: bool,
inodes_path: str,
- timeout: int):
+ timeout: int,
+ compiler_type: CompilerType):
# nothing will work if the screen isn't unlocked first.
cmd_utils.execute_arbitrary_command([_UNLOCK_SCREEN_SCRIPT],
timeout,
@@ -273,7 +283,8 @@
run_combos,
simulate,
inodes_path,
- timeout)
+ timeout,
+ compiler_type)
def gather_results(commands: Iterable[Tuple[DataFrame]],
key_list: List[str], value_list: List[Tuple[str, ...]]):
@@ -361,7 +372,8 @@
exec = execute_run_combos(grouped_combos(),
opts.simulate,
opts.inodes,
- opts.timeout)
+ opts.timeout,
+ opts.compiler_type)
results = gather_results(exec, _COMBINATORIAL_OPTIONS, combos())
diff --git a/startop/scripts/app_startup/app_startup_runner_test.py b/startop/scripts/app_startup/app_startup_runner_test.py
index 42ea5f0..382f6f3 100755
--- a/startop/scripts/app_startup/app_startup_runner_test.py
+++ b/startop/scripts/app_startup/app_startup_runner_test.py
@@ -92,7 +92,7 @@
"""
d = {'compiler_filters': None, 'simulate': False, 'debug': False,
'output': None, 'timeout': 10, 'loop_count': 1, 'inodes': None,
- 'trace_duration': None}
+ 'trace_duration': None, 'compiler_type': asr.CompilerType.HOST}
d.update(kwargs)
return d
diff --git a/startop/scripts/iorap/compiler.py b/startop/scripts/iorap/compiler.py
old mode 100755
new mode 100644
index 17b58c1..1426d34
--- a/startop/scripts/iorap/compiler.py
+++ b/startop/scripts/iorap/compiler.py
@@ -1,323 +1,73 @@
#!/usr/bin/env python3
-
#
-# Copyright (C) 2019 The Android Open Source Project
+# Copyright 2019, The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
-#
-# Dependencies:
-#
-# $> sudo apt-get install python3-pip
-# $> pip3 install --user protobuf sqlalchemy sqlite3
-#
-
-import optparse
+import importlib
import os
-import re
import sys
import tempfile
-from pathlib import Path
-from datetime import timedelta
-from typing import Iterable, Optional, List
+from enum import Enum
+from typing import TextIO, List
+# local import
DIR = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.dirname(DIR))
-from iorap.generated.TraceFile_pb2 import *
-from iorap.lib.inode2filename import Inode2Filename
+import lib.print_utils as print_utils
-parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-sys.path.append(parent_dir_name)
-from trace_analyzer.lib.trace2db import Trace2Db, MmFilemapAddToPageCache, \
- RawFtraceEntry
-import lib.cmd_utils as cmd_utils
+# Type of compiler.
+class CompilerType(Enum):
+ HOST = 1 # iorap.cmd.compiler on host
+ DEVICE = 2 # adb shell iorap.cmd.compiler
+ RI = 3 # compiler.py
-_PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes.
-ANDROID_BUILD_TOP = Path(parent_dir_name).parents[3]
-TRACECONV_BIN = ANDROID_BUILD_TOP.joinpath(
- 'external/perfetto/tools/traceconv')
+def compile_perfetto_trace_ri(
+ argv: List[str],
+ compiler) -> TextIO:
+ print_utils.debug_print('Compile using RI compiler.')
+ compiler_trace_file = tempfile.NamedTemporaryFile()
+ argv.extend(['-o', compiler_trace_file.name])
+ print_utils.debug_print(argv)
+ compiler.main([''] + argv)
+ return compiler_trace_file
-class PageRun:
- """
- Intermediate representation for a run of one or more pages.
- """
- def __init__(self, device_number: int, inode: int, offset: int, length: int):
- self.device_number = device_number
- self.inode = inode
- self.offset = offset
- self.length = length
+def compile_perfetto_trace_device(inodes_path: str,
+ package: str,
+ activity: str,
+ compiler) -> TextIO:
+ print_utils.debug_print('Compile using on-device compiler.')
+ compiler_trace_file = tempfile.NamedTemporaryFile()
+ compiler.main(inodes_path, package, activity, compiler_trace_file.name)
+ return compiler_trace_file
- def __str__(self):
- return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \
- %(self.device_number, self.inode, self.offset, self.length)
+def compile(compiler_type: CompilerType,
+ inodes_path: str,
+ ri_compiler_argv,
+ package: str,
+ activity: str) -> TextIO:
+ if compiler_type == CompilerType.RI:
+ compiler = importlib.import_module('iorap.compiler_ri')
+ compiler_trace_file = compile_perfetto_trace_ri(ri_compiler_argv,
+ compiler)
+ return compiler_trace_file
+ if compiler_type == CompilerType.DEVICE:
+ compiler = importlib.import_module('iorap.compiler_device')
+ compiler_trace_file = compile_perfetto_trace_device(inodes_path,
+ package,
+ activity,
+ compiler)
+ return compiler_trace_file
-def debug_print(msg):
- #print(msg)
- pass
-
-UNDER_LAUNCH = False
-
-def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]):
- global _PAGE_SIZE
-
- runs = [
- PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs,
- length=_PAGE_SIZE)
- for pg_entry in page_cache_entries
- ]
-
- for r in runs:
- debug_print(r)
-
- print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE))
-
- return runs
-
-def optimize_page_runs(page_runs):
- new_entries = []
- last_entry = None
- for pg_entry in page_runs:
- if last_entry:
- if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode:
- # we are dealing with a run for the same exact file as a previous run.
- if pg_entry.offset == last_entry.offset + last_entry.length:
- # trivially contiguous entries. merge them together.
- last_entry.length += pg_entry.length
- continue
- # Default: Add the run without merging it to a previous run.
- last_entry = pg_entry
- new_entries.append(pg_entry)
- return new_entries
-
-def is_filename_matching_filter(file_name, filters=[]):
- """
- Blacklist-style regular expression filters.
-
- :return: True iff file_name has an RE match in one of the filters.
- """
- for filt in filters:
- res = re.search(filt, file_name)
- if res:
- return True
-
- return False
-
-def build_protobuf(page_runs, inode2filename, filters=[]):
- trace_file = TraceFile()
- trace_file_index = trace_file.index
-
- file_id_counter = 0
- file_id_map = {} # filename -> id
-
- stats_length_total = 0
- filename_stats = {} # filename -> total size
-
- skipped_inode_map = {}
- filtered_entry_map = {} # filename -> count
-
- for pg_entry in page_runs:
- fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode)
- if not fn:
- skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1
- continue
-
- filename = fn
-
- if filters and not is_filename_matching_filter(filename, filters):
- filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1
- continue
-
- file_id = file_id_map.get(filename)
- if not file_id:
- file_id = file_id_counter
- file_id_map[filename] = file_id_counter
- file_id_counter = file_id_counter + 1
-
- file_index_entry = trace_file_index.entries.add()
- file_index_entry.id = file_id
- file_index_entry.file_name = filename
-
- # already in the file index, add the file entry.
- file_entry = trace_file.list.entries.add()
- file_entry.index_id = file_id
- file_entry.file_length = pg_entry.length
- stats_length_total += file_entry.file_length
- file_entry.file_offset = pg_entry.offset
-
- filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length
-
- for inode, count in skipped_inode_map.items():
- print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count))
-
- print("Stats: Sum of lengths %d" %(stats_length_total))
-
- if filters:
- print("Filter: %d total files removed." %(len(filtered_entry_map)))
-
- for fn, count in filtered_entry_map.items():
- print("Filter: File '%s' removed '%d' entries." %(fn, count))
-
- for filename, file_size in filename_stats.items():
- print("%s,%s" %(filename, file_size))
-
- return trace_file
-
-def calc_trace_end_time(trace2db: Trace2Db,
- trace_duration: Optional[timedelta]) -> float:
- """
- Calculates the end time based on the trace duration.
- The start time is the first receiving mm file map event.
- The end time is the start time plus the trace duration.
- All of them are in milliseconds.
- """
- # If the duration is not set, assume all time is acceptable.
- if trace_duration is None:
- # float('inf')
- return RawFtraceEntry.__table__.c.timestamp.type.python_type('inf')
-
- first_event = trace2db.session.query(MmFilemapAddToPageCache).join(
- MmFilemapAddToPageCache.raw_ftrace_entry).order_by(
- RawFtraceEntry.timestamp).first()
-
- # total_seconds() will return a float number.
- return first_event.raw_ftrace_entry.timestamp + trace_duration.total_seconds()
-
-def query_add_to_page_cache(trace2db: Trace2Db, trace_duration: Optional[timedelta]):
- end_time = calc_trace_end_time(trace2db, trace_duration)
- # SELECT * FROM tbl ORDER BY id;
- return trace2db.session.query(MmFilemapAddToPageCache).join(
- MmFilemapAddToPageCache.raw_ftrace_entry).filter(
- RawFtraceEntry.timestamp <= end_time).order_by(
- MmFilemapAddToPageCache.id).all()
-
-def transform_perfetto_trace_to_systrace(path_to_perfetto_trace: str,
- path_to_tmp_systrace: str) -> None:
- """ Transforms the systrace file from perfetto trace. """
- cmd_utils.run_command_nofail([str(TRACECONV_BIN),
- 'systrace',
- path_to_perfetto_trace,
- path_to_tmp_systrace])
-
-
-def run(sql_db_path:str,
- trace_file:str,
- trace_duration:Optional[timedelta],
- output_file:str,
- inode_table:str,
- filter:List[str]) -> int:
- trace2db = Trace2Db(sql_db_path)
- # Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache.
- trace2db.set_raw_ftrace_entry_filter(\
- lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache')
- # TODO: parse multiple trace files here.
- parse_count = trace2db.parse_file_into_db(trace_file)
-
- mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db,
- trace_duration)
- print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows)))
-
- page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows)
- print("DONE. Converted %d entries" %(len(page_runs)))
-
- # TODO: flags to select optimizations.
- optimized_page_runs = optimize_page_runs(page_runs)
- print("DONE. Optimized down to %d entries" %(len(optimized_page_runs)))
-
- print("Build protobuf...")
- trace_file = build_protobuf(optimized_page_runs, inode_table, filter)
-
- print("Write protobuf to file...")
- output_file = open(output_file, 'wb')
- output_file.write(trace_file.SerializeToString())
- output_file.close()
-
- print("DONE")
-
- # TODO: Silent running mode [no output except on error] for build runs.
-
- return 0
-
-def main(argv):
- parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb")
- parser.add_option('-i', dest='inode_data_file', metavar='FILE',
- help='Read cached inode data from a file saved earlier with pagecache.py -d')
- parser.add_option('-t', dest='trace_file', metavar='FILE',
- help='Path to systrace file (trace.html) that will be parsed')
- parser.add_option('--perfetto-trace', dest='perfetto_trace_file',
- metavar='FILE',
- help='Path to perfetto trace that will be parsed')
-
- parser.add_option('--db', dest='sql_db', metavar='FILE',
- help='Path to intermediate sqlite3 database [default: in-memory].')
-
- parser.add_option('-f', dest='filter', action="append", default=[],
- help="Add file filter. All file entries not matching one of the filters are discarded.")
-
- parser.add_option('-l', dest='launch_lock', action="store_true", default=False,
- help="Exclude all events not inside launch_lock")
-
- parser.add_option('-o', dest='output_file', metavar='FILE',
- help='Output protobuf file')
-
- parser.add_option('--duration', dest='trace_duration', action="store",
- type=int, help='The duration of trace in milliseconds.')
-
- options, categories = parser.parse_args(argv[1:])
-
- # TODO: OptionParser should have some flags to make these mandatory.
- if not options.inode_data_file:
- parser.error("-i is required")
- if not options.trace_file and not options.perfetto_trace_file:
- parser.error("one of -t or --perfetto-trace is required")
- if options.trace_file and options.perfetto_trace_file:
- parser.error("please enter either -t or --perfetto-trace, not both")
- if not options.output_file:
- parser.error("-o is required")
-
- if options.launch_lock:
- print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.")
-
- inode_table = Inode2Filename.new_from_filename(options.inode_data_file)
-
- sql_db_path = ":memory:"
- if options.sql_db:
- sql_db_path = options.sql_db
-
- trace_duration = timedelta(milliseconds=options.trace_duration) if \
- options.trace_duration is not None else None
-
- # if the input is systrace
- if options.trace_file:
- return run(sql_db_path,
- options.trace_file,
- trace_duration,
- options.output_file,
- inode_table,
- options.filter)
-
- # if the input is perfetto trace
- # TODO python 3.7 switch to using nullcontext
- with tempfile.NamedTemporaryFile() as trace_file:
- transform_perfetto_trace_to_systrace(options.perfetto_trace_file,
- trace_file.name)
- return run(sql_db_path,
- trace_file.name,
- trace_duration,
- options.output_file,
- inode_table,
- options.filter)
-
-if __name__ == '__main__':
- print(sys.argv)
- sys.exit(main(sys.argv))
+ # Should not arrive here.
+ raise ValueError('Unknown compiler type')
diff --git a/startop/scripts/iorap/compiler_device.py b/startop/scripts/iorap/compiler_device.py
new file mode 100644
index 0000000..d941cd9
--- /dev/null
+++ b/startop/scripts/iorap/compiler_device.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import sys
+from typing import List
+
+DIR = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.dirname(DIR)) # framework/base/startop/script
+import lib.print_utils as print_utils
+import iorap.lib.iorapd_utils as iorapd_utils
+from app_startup.lib.app_runner import AppRunner
+
+IORAP_COMMON_BASH_SCRIPT = os.path.join(DIR, 'common')
+
+def parse_options(argv: List[str] = None):
+ """Parses command line arguments and returns an argparse Namespace object."""
+ parser = argparse.ArgumentParser(description="Compile perfetto trace file")
+ required_named = parser.add_argument_group('required named arguments')
+
+ required_named.add_argument('-i', dest='inodes', metavar='FILE',
+ help='Read cached inode data from a file saved '
+ 'earlier with pagecache.py -d')
+ required_named.add_argument('-p', dest='package',
+ help='Package of the app to be compiled')
+
+ optional_named = parser.add_argument_group('optional named arguments')
+ optional_named.add_argument('-o', dest='output',
+ help='The compiled trace is stored into the output file')
+ optional_named.add_argument('-a', dest='activity',
+ help='Activity of the app to be compiled')
+ optional_named.add_argument('-d', dest='debug', action='store_true'
+ , help='Activity of the app to be compiled')
+
+ return parser.parse_args(argv)
+
+def main(inodes, package, activity, output, **kwargs) -> int:
+ """Entries of the program."""
+ if not activity:
+ activity = AppRunner.get_activity(package)
+
+ passed = iorapd_utils.compile_perfetto_trace_on_device(package, activity,
+ inodes)
+ if passed and output:
+ iorapd_utils.get_iorapd_compiler_trace(package, activity, output)
+
+ return 0
+
+if __name__ == '__main__':
+ opts = parse_options()
+ if opts.debug:
+ print_utils.DEBUG = opts.debug
+ print_utils.debug_print(opts)
+ sys.exit(main(**(vars(opts))))
diff --git a/startop/scripts/iorap/compiler_ri.py b/startop/scripts/iorap/compiler_ri.py
new file mode 100755
index 0000000..17b58c1
--- /dev/null
+++ b/startop/scripts/iorap/compiler_ri.py
@@ -0,0 +1,323 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Dependencies:
+#
+# $> sudo apt-get install python3-pip
+# $> pip3 install --user protobuf sqlalchemy sqlite3
+#
+
+import optparse
+import os
+import re
+import sys
+import tempfile
+from pathlib import Path
+from datetime import timedelta
+from typing import Iterable, Optional, List
+
+DIR = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.dirname(DIR))
+from iorap.generated.TraceFile_pb2 import *
+from iorap.lib.inode2filename import Inode2Filename
+
+parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+sys.path.append(parent_dir_name)
+from trace_analyzer.lib.trace2db import Trace2Db, MmFilemapAddToPageCache, \
+ RawFtraceEntry
+import lib.cmd_utils as cmd_utils
+
+_PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes.
+ANDROID_BUILD_TOP = Path(parent_dir_name).parents[3]
+TRACECONV_BIN = ANDROID_BUILD_TOP.joinpath(
+ 'external/perfetto/tools/traceconv')
+
+class PageRun:
+ """
+ Intermediate representation for a run of one or more pages.
+ """
+ def __init__(self, device_number: int, inode: int, offset: int, length: int):
+ self.device_number = device_number
+ self.inode = inode
+ self.offset = offset
+ self.length = length
+
+ def __str__(self):
+ return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \
+ %(self.device_number, self.inode, self.offset, self.length)
+
+def debug_print(msg):
+ #print(msg)
+ pass
+
+UNDER_LAUNCH = False
+
+def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]):
+ global _PAGE_SIZE
+
+ runs = [
+ PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs,
+ length=_PAGE_SIZE)
+ for pg_entry in page_cache_entries
+ ]
+
+ for r in runs:
+ debug_print(r)
+
+ print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE))
+
+ return runs
+
+def optimize_page_runs(page_runs):
+ new_entries = []
+ last_entry = None
+ for pg_entry in page_runs:
+ if last_entry:
+ if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode:
+ # we are dealing with a run for the same exact file as a previous run.
+ if pg_entry.offset == last_entry.offset + last_entry.length:
+ # trivially contiguous entries. merge them together.
+ last_entry.length += pg_entry.length
+ continue
+ # Default: Add the run without merging it to a previous run.
+ last_entry = pg_entry
+ new_entries.append(pg_entry)
+ return new_entries
+
+def is_filename_matching_filter(file_name, filters=[]):
+ """
+ Blacklist-style regular expression filters.
+
+ :return: True iff file_name has an RE match in one of the filters.
+ """
+ for filt in filters:
+ res = re.search(filt, file_name)
+ if res:
+ return True
+
+ return False
+
+def build_protobuf(page_runs, inode2filename, filters=[]):
+ trace_file = TraceFile()
+ trace_file_index = trace_file.index
+
+ file_id_counter = 0
+ file_id_map = {} # filename -> id
+
+ stats_length_total = 0
+ filename_stats = {} # filename -> total size
+
+ skipped_inode_map = {}
+ filtered_entry_map = {} # filename -> count
+
+ for pg_entry in page_runs:
+ fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode)
+ if not fn:
+ skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1
+ continue
+
+ filename = fn
+
+ if filters and not is_filename_matching_filter(filename, filters):
+ filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1
+ continue
+
+ file_id = file_id_map.get(filename)
+ if not file_id:
+ file_id = file_id_counter
+ file_id_map[filename] = file_id_counter
+ file_id_counter = file_id_counter + 1
+
+ file_index_entry = trace_file_index.entries.add()
+ file_index_entry.id = file_id
+ file_index_entry.file_name = filename
+
+ # already in the file index, add the file entry.
+ file_entry = trace_file.list.entries.add()
+ file_entry.index_id = file_id
+ file_entry.file_length = pg_entry.length
+ stats_length_total += file_entry.file_length
+ file_entry.file_offset = pg_entry.offset
+
+ filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length
+
+ for inode, count in skipped_inode_map.items():
+ print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count))
+
+ print("Stats: Sum of lengths %d" %(stats_length_total))
+
+ if filters:
+ print("Filter: %d total files removed." %(len(filtered_entry_map)))
+
+ for fn, count in filtered_entry_map.items():
+ print("Filter: File '%s' removed '%d' entries." %(fn, count))
+
+ for filename, file_size in filename_stats.items():
+ print("%s,%s" %(filename, file_size))
+
+ return trace_file
+
+def calc_trace_end_time(trace2db: Trace2Db,
+ trace_duration: Optional[timedelta]) -> float:
+ """
+ Calculates the end time based on the trace duration.
+ The start time is the first receiving mm file map event.
+ The end time is the start time plus the trace duration.
+ All of them are in milliseconds.
+ """
+ # If the duration is not set, assume all time is acceptable.
+ if trace_duration is None:
+ # float('inf')
+ return RawFtraceEntry.__table__.c.timestamp.type.python_type('inf')
+
+ first_event = trace2db.session.query(MmFilemapAddToPageCache).join(
+ MmFilemapAddToPageCache.raw_ftrace_entry).order_by(
+ RawFtraceEntry.timestamp).first()
+
+ # total_seconds() will return a float number.
+ return first_event.raw_ftrace_entry.timestamp + trace_duration.total_seconds()
+
+def query_add_to_page_cache(trace2db: Trace2Db, trace_duration: Optional[timedelta]):
+ end_time = calc_trace_end_time(trace2db, trace_duration)
+ # SELECT * FROM tbl ORDER BY id;
+ return trace2db.session.query(MmFilemapAddToPageCache).join(
+ MmFilemapAddToPageCache.raw_ftrace_entry).filter(
+ RawFtraceEntry.timestamp <= end_time).order_by(
+ MmFilemapAddToPageCache.id).all()
+
+def transform_perfetto_trace_to_systrace(path_to_perfetto_trace: str,
+ path_to_tmp_systrace: str) -> None:
+ """ Transforms the systrace file from perfetto trace. """
+ cmd_utils.run_command_nofail([str(TRACECONV_BIN),
+ 'systrace',
+ path_to_perfetto_trace,
+ path_to_tmp_systrace])
+
+
+def run(sql_db_path:str,
+ trace_file:str,
+ trace_duration:Optional[timedelta],
+ output_file:str,
+ inode_table:str,
+ filter:List[str]) -> int:
+ trace2db = Trace2Db(sql_db_path)
+ # Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache.
+ trace2db.set_raw_ftrace_entry_filter(\
+ lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache')
+ # TODO: parse multiple trace files here.
+ parse_count = trace2db.parse_file_into_db(trace_file)
+
+ mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db,
+ trace_duration)
+ print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows)))
+
+ page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows)
+ print("DONE. Converted %d entries" %(len(page_runs)))
+
+ # TODO: flags to select optimizations.
+ optimized_page_runs = optimize_page_runs(page_runs)
+ print("DONE. Optimized down to %d entries" %(len(optimized_page_runs)))
+
+ print("Build protobuf...")
+ trace_file = build_protobuf(optimized_page_runs, inode_table, filter)
+
+ print("Write protobuf to file...")
+ output_file = open(output_file, 'wb')
+ output_file.write(trace_file.SerializeToString())
+ output_file.close()
+
+ print("DONE")
+
+ # TODO: Silent running mode [no output except on error] for build runs.
+
+ return 0
+
+def main(argv):
+ parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb")
+ parser.add_option('-i', dest='inode_data_file', metavar='FILE',
+ help='Read cached inode data from a file saved earlier with pagecache.py -d')
+ parser.add_option('-t', dest='trace_file', metavar='FILE',
+ help='Path to systrace file (trace.html) that will be parsed')
+ parser.add_option('--perfetto-trace', dest='perfetto_trace_file',
+ metavar='FILE',
+ help='Path to perfetto trace that will be parsed')
+
+ parser.add_option('--db', dest='sql_db', metavar='FILE',
+ help='Path to intermediate sqlite3 database [default: in-memory].')
+
+ parser.add_option('-f', dest='filter', action="append", default=[],
+ help="Add file filter. All file entries not matching one of the filters are discarded.")
+
+ parser.add_option('-l', dest='launch_lock', action="store_true", default=False,
+ help="Exclude all events not inside launch_lock")
+
+ parser.add_option('-o', dest='output_file', metavar='FILE',
+ help='Output protobuf file')
+
+ parser.add_option('--duration', dest='trace_duration', action="store",
+ type=int, help='The duration of trace in milliseconds.')
+
+ options, categories = parser.parse_args(argv[1:])
+
+ # TODO: OptionParser should have some flags to make these mandatory.
+ if not options.inode_data_file:
+ parser.error("-i is required")
+ if not options.trace_file and not options.perfetto_trace_file:
+ parser.error("one of -t or --perfetto-trace is required")
+ if options.trace_file and options.perfetto_trace_file:
+ parser.error("please enter either -t or --perfetto-trace, not both")
+ if not options.output_file:
+ parser.error("-o is required")
+
+ if options.launch_lock:
+ print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.")
+
+ inode_table = Inode2Filename.new_from_filename(options.inode_data_file)
+
+ sql_db_path = ":memory:"
+ if options.sql_db:
+ sql_db_path = options.sql_db
+
+ trace_duration = timedelta(milliseconds=options.trace_duration) if \
+ options.trace_duration is not None else None
+
+ # if the input is systrace
+ if options.trace_file:
+ return run(sql_db_path,
+ options.trace_file,
+ trace_duration,
+ options.output_file,
+ inode_table,
+ options.filter)
+
+ # if the input is perfetto trace
+ # TODO python 3.7 switch to using nullcontext
+ with tempfile.NamedTemporaryFile() as trace_file:
+ transform_perfetto_trace_to_systrace(options.perfetto_trace_file,
+ trace_file.name)
+ return run(sql_db_path,
+ trace_file.name,
+ trace_duration,
+ options.output_file,
+ inode_table,
+ options.filter)
+
+if __name__ == '__main__':
+ print(sys.argv)
+ sys.exit(main(sys.argv))
diff --git a/startop/scripts/iorap/compiler_test.py b/startop/scripts/iorap/compiler_test.py
index 1a9f059..d1f11c5 100644
--- a/startop/scripts/iorap/compiler_test.py
+++ b/startop/scripts/iorap/compiler_test.py
@@ -30,7 +30,7 @@
"""
import os
-import compiler
+import compiler_host as compiler
DIR = os.path.abspath(os.path.dirname(__file__))
TEXTCACHE = os.path.join(DIR, 'test_fixtures/compiler/common_textcache')
diff --git a/startop/scripts/iorap/lib/iorapd_utils.py b/startop/scripts/iorap/lib/iorapd_utils.py
index 0d62180..f6f21fd 100644
--- a/startop/scripts/iorap/lib/iorapd_utils.py
+++ b/startop/scripts/iorap/lib/iorapd_utils.py
@@ -18,10 +18,9 @@
import os
import sys
-from pathlib import Path
-# up to two level, like '../../'
-sys.path.append(Path(os.path.abspath(__file__)).parents[2])
+# up to two level
+sys.path.append(os.path.join(os.path.abspath(__file__),'../..'))
import lib.cmd_utils as cmd_utils
IORAPID_LIB_DIR = os.path.abspath(os.path.dirname(__file__))
@@ -39,6 +38,22 @@
# Match logic of 'AppComponentName' in iorap::compiler C++ code.
return '{}/{}%2F{}.{}'.format(IORAPD_DATA_PATH, package, activity, suffix)
+def compile_perfetto_trace_on_device(package: str, activity: str,
+ inodes: str) -> bool:
+ """Compiles the perfetto trace using on-device compiler."""
+ passed, _ = cmd_utils.run_shell_func(IORAP_COMMON_BASH_SCRIPT,
+ 'iorapd_compiler_for_app_trace',
+ [package, activity, inodes])
+ return passed
+
+def get_iorapd_compiler_trace(package: str, activity: str, dest: str) -> str:
+ """Gets compiler trace to dest file."""
+ src = _iorapd_path_to_data_file(package, activity, 'compiled_trace.pb')
+ passed, _ = cmd_utils.run_shell_command('adb pull "{}" "{}"'.format(src, dest))
+ if not passed:
+ return False
+ return True
+
def iorapd_compiler_install_trace_file(package: str, activity: str,
input_file: str) -> bool:
"""Installs a compiled trace file.