blob: 2b99359d3f9d044c745e646acf363e208850f9ac [file] [log] [blame]
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Android Workloads Experiments"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import logging\n",
"reload(logging)\n",
"log_fmt = '%(asctime)-9s %(levelname)-8s: %(message)s'\n",
"logging.basicConfig(format=log_fmt)\n",
"\n",
"# Change to info once the notebook runs ok\n",
"logging.getLogger().setLevel(logging.INFO)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"%pylab inline\n",
"\n",
"import collections\n",
"import copy\n",
"import json\n",
"import os\n",
"import pexpect as pe\n",
"from time import sleep\n",
"\n",
"# Support to access the remote target\n",
"import devlib\n",
"from env import TestEnv\n",
"\n",
"# from devlib.utils.android import adb_command\n",
"\n",
"# Import support for Android devices\n",
"from android import Screen, Workload\n",
"\n",
"# Support for trace events analysis\n",
"from trace import Trace\n",
"\n",
"# Suport for FTrace events parsing and visualization\n",
"import trappy\n",
"\n",
"import datetime"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Test Environment set up"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Devlib requires the ANDROID_HOME environment variable configured to point to your local installation of the Android SDK. If you have not this variable configured in the shell used to start the notebook server, you need to run the next cell to define where your Android SDK is installed."
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Setup Androd SDK\n",
"os.environ['ANDROID_HOME'] = '/home/eas/Work/Android/android-sdk-linux/'\n",
"\n",
"# Setup Catapult for Systrace usage\n",
"CATAPULT_HOME = \"/home/eas/Work/Android/catapult\""
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"adbd is already running as root\r\n"
]
}
],
"source": [
"# Android device to target\n",
"DEVICE = 'GA0113TP0178'\n",
"\n",
"# Ensure ADB is running as root\n",
"!adb -s {DEVICE} root"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"In case more than one Android device are conencted to the host, you must specify the ID of the device you want to target in `my_target_conf`. Run `adb devices` on your host to get the ID."
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"# Setup target configuration\n",
"my_conf = {\n",
"\n",
" # Target platform and board\n",
" \"platform\" : 'android',\n",
" \"device\" : DEVICE,\n",
"\n",
"# \"emeter\" : {\n",
"# \"instrument\" : \"aep\",\n",
"# \"conf\" : {\n",
"# 'labels' : ['BAT'],\n",
"# 'resistor_values' : [0.099],\n",
"# 'device_entry' : '/dev/ttyACM1',\n",
"# }\n",
"# },\n",
"\n",
" # Folder where all the results will be collected\n",
" \"results_dir\" : \"Android_Workloads\",\n",
"\n",
" # Define devlib modules to load\n",
" \"modules\" : [\n",
" 'cpufreq' # enable CPUFreq support\n",
" ],\n",
"\n",
" # FTrace events to collect for all the tests configuration which have\n",
" # the \"ftrace\" flag enabled\n",
" \"ftrace\" : {\n",
" \"events\" : [\n",
" \"sched_switch\",\n",
" \"sched_overutilized\",\n",
" \"sched_contrib_scale_f\",\n",
" \"sched_load_avg_cpu\",\n",
" \"sched_load_avg_task\",\n",
" \"sched_tune_tasks_update\",\n",
" \"sched_boost_cpu\",\n",
" \"sched_boost_task\",\n",
" \"sched_energy_diff\",\n",
" \"cpu_frequency\",\n",
" \"cpu_idle\",\n",
" \"cpu_capacity\",\n",
" ],\n",
" \"buffsize\" : 10 * 1024,\n",
" },\n",
"\n",
" # Tools required by the experiments\n",
" \"tools\" : [ 'trace-cmd' ],\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"# List of configurations to test (keys of 'confs' defined in cell #9)\n",
"test_confs = ['std']\n",
"\n",
"# List of workloads to run, each workload consists of a workload name\n",
"# followed by a list of workload specific parameters\n",
"test_wloads = [\n",
" \n",
"\n",
" # YouTube workload:\n",
"# Params:\n",
"# - video URL (with optional start time)\n",
"# - duration [s] to playback\n",
" 'YouTube https://youtu.be/XSGBVzeBUbk?t=45s 15',\n",
"\n",
"# Jankbench workload:\n",
"# Params:\n",
"# - id of the benchmakr to run\n",
" 'Jankbench list_view',\n",
"# 'Jankbench image_list_view',\n",
"# 'Jankbench shadow_grid',\n",
"# 'Jankbench low_hitrate_text',\n",
"# 'Jankbench high_hitrate_text',\n",
"# 'Jankbench edit_text',\n",
"\n",
"]\n",
"\n",
"# Iterations for each test\n",
"iterations = 1"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Define what we want to collect as a list of strings.\n",
"# Supported values are\n",
"# energy - Use the my_conf's defined emeter to measure energy consumption across experiments\n",
"# ftrace - Collect an execution trace using trace-cmd\n",
"# systrace - Collect an execution trace using Systrace/Atrace\n",
"# NOTE: energy is automatically enabled in case an \"emeter\" configuration is defined in my_conf\n",
"collect = ''"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Support Functions"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This set of support functions will help us running the benchmark using different CPUFreq governors."
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def set_performance():\n",
" target.cpufreq.set_all_governors('performance')\n",
"\n",
"def set_powersave():\n",
" target.cpufreq.set_all_governors('powersave')\n",
"\n",
"def set_interactive():\n",
" target.cpufreq.set_all_governors('interactive')\n",
"\n",
"def set_sched():\n",
" target.cpufreq.set_all_governors('sched')\n",
"\n",
"def set_ondemand():\n",
" target.cpufreq.set_all_governors('ondemand')\n",
" \n",
" for cpu in target.list_online_cpus():\n",
" tunables = target.cpufreq.get_governor_tunables(cpu)\n",
" target.cpufreq.set_governor_tunables(\n",
" cpu,\n",
" 'ondemand',\n",
" **{'sampling_rate' : tunables['sampling_rate_min']}\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Available test configurations\n",
"confs = {\n",
" 'std' : {\n",
" 'label' : 'int',\n",
" 'set' : set_interactive,\n",
" },\n",
" 'eas' : {\n",
" 'label' : 'sch',\n",
" 'set' : set_sched,\n",
" }\n",
"}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Experiments Execution Function"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"SYSTRACE_CMD = CATAPULT_HOME + \"/systrace/systrace/systrace.py -e {} -o {} gfx view sched freq idle -t {}\"\n",
"\n",
"def experiment(wl, res_dir, conf_name, wload_name, iterations, collect=''):\n",
" \n",
" # Load workload params\n",
" wload_kind = wload_name.split()[0]\n",
" wload_tag = wload_name.split()[1]\\\n",
" .replace('https://youtu.be/', '')\\\n",
" .replace('?t=', '_')\n",
" \n",
" # Check for workload being available\n",
" wload = Workload.get(te, wload_kind)\n",
" if not wload:\n",
" return {}\n",
" \n",
" # Setup test results folder\n",
" exp_dir = os.path.join(res_dir, conf_name, \"{}_{}\".format(wload_kind, wload_tag))\n",
" os.system('mkdir -p {}'.format(exp_dir));\n",
"\n",
" # Configure governor\n",
" confs[conf_name]['set']()\n",
" \n",
" # Configure screen to max brightness and no dimming\n",
" Screen.set_brightness(target, percent=100)\n",
" Screen.set_dim(target, auto=False)\n",
" Screen.set_timeout(target, 60*60*10) # 10 hours should be enought for an experiment\n",
" \n",
" # Start the required tracing command\n",
" if 'ftrace' in collect:\n",
" # Start FTrace and Energy monitoring\n",
" te.ftrace.start()\n",
" elif 'systrace' in collect:\n",
" # Start systrace\n",
" trace_file = os.path.join(exp_dir, 'trace.html')\n",
" trace_cmd = SYSTRACE_CMD.format(DEVICE, trace_file, 10 * iterations)\n",
" logging.info('SysTrace: %s', trace_cmd)\n",
" systrace_output = pe.spawn(trace_cmd)\n",
" \n",
" ###########################\n",
" # Run the required workload\n",
" \n",
" # Jankbench\n",
" if 'Jankbench' in wload_name:\n",
" db_file, nrg_data, nrg_file = wload.run(exp_dir, wload_tag, iterations, collect)\n",
" \n",
" # YouTube\n",
" elif 'YouTube' in wload_name:\n",
" video_url = wload_name.split()[1]\n",
" video_duration_s = wload_name.split()[2]\n",
" db_file, nrg_data, nrg_file = wload.run(exp_dir, video_url, int(video_duration_s), collect)\n",
"\n",
" ###########################\n",
" \n",
" # Stop the required trace command\n",
" if 'ftrace' in collect:\n",
" te.ftrace.stop()\n",
" # Collect and keep track of the trace\n",
" trace_file = os.path.join(exp_dir, 'trace.dat')\n",
" te.ftrace.get_trace(trace_file)\n",
" elif 'systrace' in collect:\n",
" logging.info('Waiting systrace report [%s]...', trace_file)\n",
" systrace_output.wait()\n",
"\n",
" # Reset screen brightness and auto dimming\n",
" Screen.set_defaults(target, )\n",
" \n",
" # Dump platform descriptor\n",
" te.platform_dump(exp_dir)\n",
"\n",
" # return all the experiment data\n",
" if 'trace' in collect:\n",
" return {\n",
" 'dir' : exp_dir,\n",
" 'db_file' : db_file,\n",
" 'nrg_data' : copy.deepcopy(nrg_data),\n",
" 'nrg_file' : nrg_file,\n",
" 'trace_file' : trace_file,\n",
" }\n",
" else:\n",
" return {\n",
" 'dir' : exp_dir,\n",
" 'db_file' : db_file,\n",
" 'nrg_data' : copy.deepcopy(nrg_data),\n",
" 'nrg_file' : nrg_file,\n",
" }"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Main"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Target Connection"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"# # Cleanup Caiman energy meter temporary folders\n",
"# !rm -rf /tmp/eprobe-caiman-*\n",
"# # Ensure there are not other \"caiman\" instanced running for the specified device\n",
"# # my_conf['emeter']['conf']['device_entry']\n",
"# !killall caiman"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2016-05-10 18:11:10,176 INFO : Target - Using base path: /home/derkling/Code/lisa\n",
"2016-05-10 18:11:10,177 INFO : Target - Loading custom (inline) target configuration\n",
"2016-05-10 18:11:10,178 INFO : Target - Devlib modules to load: ['cpufreq']\n",
"2016-05-10 18:11:10,179 INFO : Target - Connecting Android target [GA0113TP0178]\n",
"2016-05-10 18:11:11,481 INFO : Target - Initializing target workdir:\n",
"2016-05-10 18:11:11,482 INFO : Target - /data/local/tmp/devlib-target\n",
"2016-05-10 18:11:14,396 INFO : Target - Topology:\n",
"2016-05-10 18:11:14,397 INFO : Target - [[0, 1], [2, 3]]\n",
"2016-05-10 18:11:15,643 WARNING : Event [sched_overutilized] not available for tracing\n",
"2016-05-10 18:11:15,645 WARNING : Event [sched_contrib_scale_f] not available for tracing\n",
"2016-05-10 18:11:15,647 WARNING : Event [sched_load_avg_cpu] not available for tracing\n",
"2016-05-10 18:11:15,648 WARNING : Event [sched_load_avg_task] not available for tracing\n",
"2016-05-10 18:11:15,649 WARNING : Event [sched_tune_tasks_update] not available for tracing\n",
"2016-05-10 18:11:15,651 WARNING : Event [sched_boost_cpu] not available for tracing\n",
"2016-05-10 18:11:15,652 WARNING : Event [sched_boost_task] not available for tracing\n",
"2016-05-10 18:11:15,654 WARNING : Event [sched_energy_diff] not available for tracing\n",
"2016-05-10 18:11:15,657 WARNING : Event [cpu_capacity] not available for tracing\n",
"2016-05-10 18:11:15,657 INFO : FTrace - Enabled tracepoints:\n",
"2016-05-10 18:11:15,658 INFO : FTrace - sched_switch\n",
"2016-05-10 18:11:15,658 INFO : FTrace - sched_overutilized\n",
"2016-05-10 18:11:15,659 INFO : FTrace - sched_contrib_scale_f\n",
"2016-05-10 18:11:15,660 INFO : FTrace - sched_load_avg_cpu\n",
"2016-05-10 18:11:15,660 INFO : FTrace - sched_load_avg_task\n",
"2016-05-10 18:11:15,661 INFO : FTrace - sched_tune_tasks_update\n",
"2016-05-10 18:11:15,661 INFO : FTrace - sched_boost_cpu\n",
"2016-05-10 18:11:15,662 INFO : FTrace - sched_boost_task\n",
"2016-05-10 18:11:15,662 INFO : FTrace - sched_energy_diff\n",
"2016-05-10 18:11:15,663 INFO : FTrace - cpu_frequency\n",
"2016-05-10 18:11:15,664 INFO : FTrace - cpu_idle\n",
"2016-05-10 18:11:15,664 INFO : FTrace - cpu_capacity\n",
"2016-05-10 18:11:15,665 INFO : TestEnv - Set results folder to:\n",
"2016-05-10 18:11:15,665 INFO : TestEnv - /home/derkling/Code/lisa/results/Android_Workloads\n",
"2016-05-10 18:11:15,666 INFO : TestEnv - Experiment results available also in:\n",
"2016-05-10 18:11:15,666 INFO : TestEnv - /home/derkling/Code/lisa/results_latest\n"
]
}
],
"source": [
"# Initialize a test environment using:\n",
"te = TestEnv(my_conf, wipe=False)\n",
"target = te.target"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Workloads Execution and Data Collection"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2016-05-10 18:11:17,190 INFO : Workload - Workloads available on target:\n",
"2016-05-10 18:11:17,192 INFO : Workload - ['YouTube', 'Jankbench']\n",
"2016-05-10 18:11:17,193 INFO : ------------------------\n",
"2016-05-10 18:11:17,194 INFO : Test 1/2: YOUTUBE in STD configuration\n",
"2016-05-10 18:11:17,195 INFO : Workload - Workloads available on target:\n",
"2016-05-10 18:11:17,195 INFO : Workload - ['YouTube', 'Jankbench']\n",
"2016-05-10 18:11:18,282 INFO : Set brightness: 100%\n",
"2016-05-10 18:11:18,829 INFO : Dim screen mode: OFF\n",
"2016-05-10 18:11:19,440 INFO : Screen timeout: 36000 [s]\n",
"2016-05-10 18:11:20,631 INFO : Force manual orientation\n",
"2016-05-10 18:11:20,632 INFO : Set orientation: LANDSCAPE\n",
"2016-05-10 18:11:21,776 INFO : am start -a android.intent.action.VIEW \"https://youtu.be/XSGBVzeBUbk?t=45s\"\n",
"2016-05-10 18:11:25,586 INFO : Play video for 15 [s]\n",
"2016-05-10 18:11:42,883 INFO : Set orientation: AUTO\n",
"2016-05-10 18:11:43,971 INFO : Set orientation: AUTO\n",
"2016-05-10 18:11:45,581 INFO : Set brightness: AUTO\n",
"2016-05-10 18:11:46,153 INFO : Dim screen mode: ON\n",
"2016-05-10 18:11:46,705 INFO : Screen timeout: 30 [s]\n",
"2016-05-10 18:11:46,707 INFO : ------------------------\n",
"2016-05-10 18:11:46,708 INFO : Test 2/2: JANKBENCH in STD configuration\n",
"2016-05-10 18:11:46,708 INFO : Workload - Workloads available on target:\n",
"2016-05-10 18:11:46,709 INFO : Workload - ['YouTube', 'Jankbench']\n",
"2016-05-10 18:11:47,841 INFO : Set brightness: 100%\n",
"2016-05-10 18:11:48,361 INFO : Dim screen mode: OFF\n",
"2016-05-10 18:11:48,971 INFO : Screen timeout: 36000 [s]\n",
"2016-05-10 18:11:51,289 INFO : Force manual orientation\n",
"2016-05-10 18:11:51,290 INFO : Set orientation: PORTRAIT\n",
"2016-05-10 18:11:52,543 INFO : am start -n \"com.android.benchmark/.app.RunLocalBenchmarksActivity\" --eia \"com.android.benchmark.EXTRA_ENABLED_BENCHMARK_IDS\" 0 --ei \"com.android.benchmark.EXTRA_RUN_COUNT\" 1\n",
"2016-05-10 18:11:53,186 INFO : adb -s GA0113TP0178 logcat ActivityManager:* System.out:I *:S BENCH:*\n",
"2016-05-10 18:12:28,308 INFO : Mean: 24.202 JankP: 0.061 StdDev: 21.430 Count Bad: 4 Count Jank: 1\n",
"2016-05-10 18:12:31,396 INFO : Set orientation: AUTO\n",
"2016-05-10 18:12:32,452 INFO : Set orientation: AUTO\n",
"2016-05-10 18:12:34,180 INFO : Set brightness: AUTO\n",
"2016-05-10 18:12:34,750 INFO : Dim screen mode: ON\n",
"2016-05-10 18:12:35,320 INFO : Screen timeout: 30 [s]\n"
]
}
],
"source": [
"# Unlock device screen (assume no password required)\n",
"target.execute('input keyevent 82')\n",
"\n",
"# Intialize Workloads for this test environment\n",
"wl = Workload(te)\n",
"\n",
"# The set of results for each comparison test\n",
"results = collections.defaultdict(dict)\n",
"\n",
"# Enable energy collection if an emeter has been configured\n",
"if 'emeter' in my_conf and te.emeter:\n",
" logging.info('Enabling ENERGY collection')\n",
" collect += ' energy'\n",
"\n",
"# Run the benchmark in all the configured governors\n",
"for conf_name in test_confs:\n",
"\n",
" for idx,wload_name in enumerate(test_wloads):\n",
" \n",
" wload_kind = wload_name.split()[0]\n",
" logging.info('------------------------')\n",
" logging.info('Test %d/%d: %s in %s configuration',\n",
" idx+1, len(test_wloads), wload_kind.upper(), conf_name.upper())\n",
" res = experiment(wl, te.res_dir, conf_name, wload_name, iterations, collect)\n",
" results[conf_name][wload_name] = res\n",
"\n",
" # Save collected results\n",
" res_file = os.path.join(te.res_dir, conf_name, 'results.json')\n",
" with open(res_file, 'w') as fh:\n",
" json.dump(results[conf_name], fh, indent=4)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Energy Measurements Report"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Energy consumption STD, YOUTUBE HTTPS://YOUTU.BE/XSGBVZEBUBK?T=45S 15 : NaN\n",
"Energy consumption STD, JANKBENCH LIST_VIEW : NaN\n"
]
}
],
"source": [
"for conf_name in test_confs:\n",
" for idx,wload_name in enumerate(test_wloads):\n",
" nrg = 'NaN'\n",
" if results[conf_name][wload_name]['nrg_data']:\n",
" nrg = '{:6.1f}'.format(float(results[conf_name][wload_name]['nrg_data']['BAT']))\n",
" print \"Energy consumption {}, {:52}: {}\".format(conf_name.upper(), wload_name.upper(), nrg)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.9"
},
"toc": {
"toc_cell": false,
"toc_number_sections": true,
"toc_threshold": 6,
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 0
}