blob: af1b765ee5c1015dcd0e144356359c6bf8928b58 [file] [log] [blame]
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Android Workloads Experiments"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2016-12-06 19:28:40,088 INFO : root : Using LISA logging configuration:\n",
"2016-12-06 19:28:40,089 INFO : root : /home/vagrant/lisa/logging.conf\n"
]
}
],
"source": [
"import logging\n",
"from conf import LisaLogging\n",
"LisaLogging.setup()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"%pylab inline\n",
"\n",
"import collections\n",
"import copy\n",
"import json\n",
"import os\n",
"from time import sleep\n",
"\n",
"# Support to access the remote target\n",
"import devlib\n",
"from env import TestEnv\n",
"\n",
"# from devlib.utils.android import adb_command\n",
"\n",
"# Import support for Android devices\n",
"from android import Screen, Workload, System\n",
"\n",
"# Support for trace events analysis\n",
"from trace import Trace\n",
"\n",
"# Suport for FTrace events parsing and visualization\n",
"import trappy\n",
"\n",
"import datetime"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Test Environment set up"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Devlib requires the ANDROID_HOME environment variable configured to point to your local installation of the Android SDK. If you have not this variable configured in the shell used to start the notebook server, you need to run an additional cell to define where your Android SDK is installed or add \"ANDROID_HOME\" in your target configuration.\n",
"CATAPULT_HOME is considered to be in LISA_HOME/tools/catapult."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"In case more than one Android device are conencted to the host, you must specify the ID of the device you want to target in `my_target_conf`. Run `adb devices` on your host to get the ID."
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"# Android device to target\n",
"DEVICE = 'HT6670300102'"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"# Setup target configuration\n",
"my_conf = {\n",
"\n",
" # Target platform and board\n",
" \"platform\" : 'android',\n",
" \"device\" : DEVICE,\n",
" \"ANDROID_HOME\" : '/home/vagrant/lisa/tools/android-sdk-linux/',\n",
"\n",
"# \"emeter\" : {\n",
"# \"instrument\" : \"aep\",\n",
"# \"conf\" : {\n",
"# 'labels' : ['BAT'],\n",
"# 'resistor_values' : [0.099],\n",
"# 'device_entry' : '/dev/ttyACM1',\n",
"# }\n",
"# },\n",
"\n",
" # Folder where all the results will be collected\n",
" \"results_dir\" : \"Android_Workloads\",\n",
"\n",
" # Define devlib modules to load\n",
" \"modules\" : [\n",
" 'cpufreq' # enable CPUFreq support\n",
" ],\n",
"\n",
" # FTrace events to collect for all the tests configuration which have\n",
" # the \"ftrace\" flag enabled\n",
" \"ftrace\" : {\n",
" \"events\" : [\n",
" \"sched_switch\",\n",
" \"sched_overutilized\",\n",
" \"sched_contrib_scale_f\",\n",
" \"sched_load_avg_cpu\",\n",
" \"sched_load_avg_task\",\n",
" \"sched_tune_tasks_update\",\n",
" \"sched_boost_cpu\",\n",
" \"sched_boost_task\",\n",
" \"sched_energy_diff\",\n",
" \"cpu_frequency\",\n",
" \"cpu_idle\",\n",
" \"cpu_capacity\",\n",
" ],\n",
" \"buffsize\" : 10 * 1024,\n",
" },\n",
"\n",
" # Tools required by the experiments\n",
" \"tools\" : [ 'trace-cmd' ],\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"# List of configurations to test (keys of 'confs' defined in cell #9)\n",
"test_confs = ['std']\n",
"\n",
"# List of workloads to run, each workload consists of a workload name\n",
"# followed by a list of workload specific parameters\n",
"test_wloads = [\n",
" \n",
"\n",
" # YouTube workload:\n",
"# Params:\n",
"# - video URL (with optional start time)\n",
"# - duration [s] to playback\n",
" 'YouTube https://youtu.be/XSGBVzeBUbk?t=45s 15',\n",
"\n",
"# Jankbench workload:\n",
"# Params:\n",
"# - id of the benchmakr to run\n",
" 'Jankbench list_view',\n",
"# 'Jankbench image_list_view',\n",
"# 'Jankbench shadow_grid',\n",
"# 'Jankbench low_hitrate_text',\n",
"# 'Jankbench high_hitrate_text',\n",
"# 'Jankbench edit_text',\n",
"\n",
"]\n",
"\n",
"# Iterations for each test\n",
"iterations = 1"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Define what we want to collect as a list of strings.\n",
"# Supported values are\n",
"# energy - Use the my_conf's defined emeter to measure energy consumption across experiments\n",
"# ftrace - Collect an execution trace using trace-cmd\n",
"# systrace - Collect an execution trace using Systrace/Atrace\n",
"# NOTE: energy is automatically enabled in case an \"emeter\" confHT6670300102iguration is defined in my_conf\n",
"collect = 'systrace'"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Support Functions"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This set of support functions will help us running the benchmark using different CPUFreq governors."
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def set_performance():\n",
" target.cpufreq.set_all_governors('performance')\n",
"\n",
"def set_powersave():\n",
" target.cpufreq.set_all_governors('powersave')\n",
"\n",
"def set_interactive():\n",
" target.cpufreq.set_all_governors('interactive')\n",
"\n",
"def set_sched():\n",
" target.cpufreq.set_all_governors('sched')\n",
"\n",
"def set_ondemand():\n",
" target.cpufreq.set_all_governors('ondemand')\n",
" \n",
" for cpu in target.list_online_cpus():\n",
" tunables = target.cpufreq.get_governor_tunables(cpu)\n",
" target.cpufreq.set_governor_tunables(\n",
" cpu,\n",
" 'ondemand',\n",
" **{'sampling_rate' : tunables['sampling_rate_min']}\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Available test configurations\n",
"confs = {\n",
" 'std' : {\n",
" 'label' : 'int',\n",
" 'set' : set_interactive,\n",
" },\n",
" 'eas' : {\n",
" 'label' : 'sch',\n",
" 'set' : set_sched,\n",
" }\n",
"}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Experiments Execution Function"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"def experiment(wl, res_dir, conf_name, wload_name, iterations, collect=''):\n",
" \n",
" # Load workload params\n",
" wload_kind = wload_name.split()[0]\n",
" wload_tag = wload_name.split()[1]\\\n",
" .replace('https://youtu.be/', '')\\\n",
" .replace('?t=', '_')\n",
" \n",
" # Check for workload being available\n",
" wload = Workload.get(te, wload_kind)\n",
" if not wload:\n",
" return {}\n",
" \n",
" # Setup test results folder\n",
" exp_dir = os.path.join(res_dir, conf_name, \"{}_{}\".format(wload_kind, wload_tag))\n",
" os.system('mkdir -p {}'.format(exp_dir));\n",
"\n",
" # Configure governor\n",
" confs[conf_name]['set']()\n",
" \n",
" # Configure screen to max brightness and no dimming\n",
" Screen.set_brightness(target, percent=100)\n",
" Screen.set_dim(target, auto=False)\n",
" Screen.set_timeout(target, 60*60*10) # 10 hours should be enought for an experiment\n",
" \n",
" # Start the required tracing command\n",
" if 'ftrace' in collect:\n",
" # Start FTrace and Energy monitoring\n",
" te.ftrace.start()\n",
" elif 'systrace' in collect:\n",
" # Start systrace\n",
" trace_file = os.path.join(exp_dir, 'trace.html')\n",
" systrace_output = System.systrace_start(te, trace_file, 10 * iterations)\n",
" \n",
" ###########################\n",
" # Run the required workload\n",
" \n",
" # Jankbench\n",
" if 'Jankbench' in wload_name:\n",
" db_file, nrg_report = wload.run(exp_dir, wload_tag, iterations, collect)\n",
" \n",
" # YouTube\n",
" elif 'YouTube' in wload_name:\n",
" video_url = wload_name.split()[1]\n",
" video_duration_s = wload_name.split()[2]\n",
" db_file, nrg_report = wload.run(exp_dir, video_url, int(video_duration_s), collect)\n",
"\n",
" ###########################\n",
" \n",
" # Stop the required trace command\n",
" if 'ftrace' in collect:\n",
" te.ftrace.stop()\n",
" # Collect and keep track of the trace\n",
" trace_file = os.path.join(exp_dir, 'trace.dat')\n",
" te.ftrace.get_trace(trace_file)\n",
" elif 'systrace' in collect:\n",
" if systrace_output:\n",
" logging.info('Waiting systrace report [%s]...', trace_file)\n",
" systrace_output.wait()\n",
" else:\n",
" logging.warning('Systrace is not running!')\n",
"\n",
" # Reset screen brightness and auto dimming\n",
" Screen.set_defaults(target, )\n",
" \n",
" # Dump platform descriptor\n",
" te.platform_dump(exp_dir)\n",
"\n",
" # return all the experiment data\n",
" if 'trace' in collect:\n",
" return {\n",
" 'dir' : exp_dir,\n",
" 'db_file' : db_file,\n",
" 'nrg_report' : copy.deepcopy(nrg_report),\n",
" 'trace_file' : trace_file,\n",
" }\n",
" else:\n",
" return {\n",
" 'dir' : exp_dir,\n",
" 'db_file' : db_file,\n",
" 'nrg_report' : copy.deepcopy(nrg_report),\n",
" }"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Main"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Target Connection"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"caiman: no process found\r\n"
]
}
],
"source": [
"# # Cleanup Caiman energy meter temporary folders\n",
"# !rm -rf /tmp/eprobe-caiman-*\n",
"# # Ensure there are not other \"caiman\" instanced running for the specified device\n",
"# # my_conf['emeter']['conf']['device_entry']\n",
"!killall caiman"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"adbd is already running as root\r\n"
]
}
],
"source": [
"# Ensure ADB is running as root\n",
"!adb -s {DEVICE} root"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2016-12-06 19:28:52,064 INFO : TestEnv : Using base path: /home/vagrant/lisa\n",
"2016-12-06 19:28:52,065 INFO : TestEnv : Loading custom (inline) target configuration\n",
"2016-12-06 19:28:52,065 INFO : TestEnv : External tools using:\n",
"2016-12-06 19:28:52,065 INFO : TestEnv : ANDROID_HOME: /home/vagrant/lisa/tools/android-sdk-linux/\n",
"2016-12-06 19:28:52,066 INFO : TestEnv : CATAPULT_HOME: /home/vagrant/lisa/tools/catapult\n",
"2016-12-06 19:28:52,066 INFO : TestEnv : Devlib modules to load: ['cpufreq']\n",
"2016-12-06 19:28:52,066 INFO : TestEnv : Connecting Android target [HT6670300102]\n",
"2016-12-06 19:28:52,067 INFO : TestEnv : Connection settings:\n",
"2016-12-06 19:28:52,067 INFO : TestEnv : {'device': 'HT6670300102'}\n",
"2016-12-06 19:28:52,146 INFO : android : ls command is set to ls -1\n",
"2016-12-06 19:28:52,979 INFO : TestEnv : Initializing target workdir:\n",
"2016-12-06 19:28:52,982 INFO : TestEnv : /data/local/tmp/devlib-target\n",
"2016-12-06 19:28:54,953 INFO : TestEnv : Topology:\n",
"2016-12-06 19:28:54,956 INFO : TestEnv : [[0, 1], [2, 3]]\n",
"2016-12-06 19:28:55,857 INFO : TestEnv : Enabled tracepoints:\n",
"2016-12-06 19:28:55,857 INFO : TestEnv : sched_switch\n",
"2016-12-06 19:28:55,858 INFO : TestEnv : sched_overutilized\n",
"2016-12-06 19:28:55,858 INFO : TestEnv : sched_contrib_scale_f\n",
"2016-12-06 19:28:55,859 INFO : TestEnv : sched_load_avg_cpu\n",
"2016-12-06 19:28:55,859 INFO : TestEnv : sched_load_avg_task\n",
"2016-12-06 19:28:55,860 INFO : TestEnv : sched_tune_tasks_update\n",
"2016-12-06 19:28:55,860 INFO : TestEnv : sched_boost_cpu\n",
"2016-12-06 19:28:55,861 INFO : TestEnv : sched_boost_task\n",
"2016-12-06 19:28:55,861 INFO : TestEnv : sched_energy_diff\n",
"2016-12-06 19:28:55,862 INFO : TestEnv : cpu_frequency\n",
"2016-12-06 19:28:55,862 INFO : TestEnv : cpu_idle\n",
"2016-12-06 19:28:55,863 INFO : TestEnv : cpu_capacity\n",
"2016-12-06 19:28:55,863 INFO : TestEnv : Set results folder to:\n",
"2016-12-06 19:28:55,863 INFO : TestEnv : /home/vagrant/lisa/results/Android_Workloads\n",
"2016-12-06 19:28:55,864 INFO : TestEnv : Experiment results available also in:\n",
"2016-12-06 19:28:55,864 INFO : TestEnv : /home/vagrant/lisa/results_latest\n"
]
}
],
"source": [
"# Initialize a test environment using:\n",
"te = TestEnv(my_conf, wipe=False)\n",
"target = te.target"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Workloads Execution and Data Collection"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2016-12-06 19:29:38,412 INFO : Workload : Workloads available on target:\n",
"2016-12-06 19:29:38,416 INFO : Workload : ['YouTube', 'Jankbench']\n",
"2016-12-06 19:29:38,419 INFO : root : ------------------------\n",
"2016-12-06 19:29:38,420 INFO : root : Test 1/2: YOUTUBE in STD configuration\n",
"2016-12-06 19:29:38,421 INFO : Workload : Workloads available on target:\n",
"2016-12-06 19:29:38,423 INFO : Workload : ['YouTube', 'Jankbench']\n",
"2016-12-06 19:29:39,389 INFO : Screen : Set brightness: 100%\n",
"2016-12-06 19:29:39,873 INFO : Screen : Dim screen mode: OFF\n",
"2016-12-06 19:29:40,333 INFO : Screen : Screen timeout: 36000 [s]\n",
"2016-12-06 19:29:40,336 INFO : System : SysTrace: /home/vagrant/lisa/tools/catapult/systrace/systrace/run_systrace.py -e HT6670300102 -o /home/vagrant/lisa/results/Android_Workloads/std/YouTube_XSGBVzeBUbk_45s/trace.html gfx view sched freq idle -t 10\n",
"2016-12-06 19:29:41,826 INFO : Screen : Force manual orientation\n",
"2016-12-06 19:29:41,829 INFO : Screen : Set orientation: LANDSCAPE\n",
"2016-12-06 19:29:45,347 INFO : YouTube : Play video for 15 [s]\n",
"2016-12-06 19:30:01,445 INFO : Screen : Set orientation: AUTO\n",
"2016-12-06 19:30:02,322 INFO : root : Waiting systrace report [/home/vagrant/lisa/results/Android_Workloads/std/YouTube_XSGBVzeBUbk_45s/trace.html]...\n",
"2016-12-06 19:30:02,323 INFO : Screen : Set orientation: AUTO\n",
"2016-12-06 19:30:03,628 INFO : Screen : Set brightness: AUTO\n",
"2016-12-06 19:30:04,065 INFO : Screen : Dim screen mode: ON\n",
"2016-12-06 19:30:04,500 INFO : Screen : Screen timeout: 30 [s]\n",
"2016-12-06 19:30:04,602 INFO : root : ------------------------\n",
"2016-12-06 19:30:04,603 INFO : root : Test 2/2: JANKBENCH in STD configuration\n",
"2016-12-06 19:30:04,604 INFO : Workload : Workloads available on target:\n",
"2016-12-06 19:30:04,604 INFO : Workload : ['YouTube', 'Jankbench']\n",
"2016-12-06 19:30:05,629 INFO : Screen : Set brightness: 100%\n",
"2016-12-06 19:30:06,085 INFO : Screen : Dim screen mode: OFF\n",
"2016-12-06 19:30:06,559 INFO : Screen : Screen timeout: 36000 [s]\n",
"2016-12-06 19:30:06,562 INFO : System : SysTrace: /home/vagrant/lisa/tools/catapult/systrace/systrace/run_systrace.py -e HT6670300102 -o /home/vagrant/lisa/results/Android_Workloads/std/Jankbench_list_view/trace.html gfx view sched freq idle -t 10\n",
"2016-12-06 19:30:09,377 INFO : Screen : Force manual orientation\n",
"2016-12-06 19:30:09,378 INFO : Screen : Set orientation: PORTRAIT\n",
"2016-12-06 19:30:10,322 INFO : Jankbench : am start -n \"com.android.benchmark/.app.RunLocalBenchmarksActivity\" --eia \"com.android.benchmark.EXTRA_ENABLED_BENCHMARK_IDS\" 0 --ei \"com.android.benchmark.EXTRA_RUN_COUNT\" 1\n",
"2016-12-06 19:30:10,785 INFO : Jankbench : adb -s HT6670300102 logcat ActivityManager:* System.out:I *:S BENCH:*\n",
"2016-12-06 19:30:45,646 INFO : Jankbench : Mean: 29.949 JankP: 0.061 StdDev: 24.786 Count Bad: 2 Count Jank: 1\n",
"2016-12-06 19:30:48,273 INFO : Screen : Set orientation: AUTO\n",
"2016-12-06 19:30:50,116 INFO : root : Waiting systrace report [/home/vagrant/lisa/results/Android_Workloads/std/Jankbench_list_view/trace.html]...\n",
"2016-12-06 19:30:50,117 INFO : Screen : Set orientation: AUTO\n",
"2016-12-06 19:30:51,516 INFO : Screen : Set brightness: AUTO\n",
"2016-12-06 19:30:51,946 INFO : Screen : Dim screen mode: ON\n",
"2016-12-06 19:30:52,340 INFO : Screen : Screen timeout: 30 [s]\n"
]
}
],
"source": [
"# Unlock device screen (assume no password required)\n",
"target.execute('input keyevent 82')\n",
"\n",
"# Intialize Workloads for this test environment\n",
"wl = Workload(te)\n",
"\n",
"# The set of results for each comparison test\n",
"results = collections.defaultdict(dict)\n",
"\n",
"# Enable energy collection if an emeter has been configured\n",
"if 'emeter' in my_conf and te.emeter:\n",
" logging.info('Enabling ENERGY collection')\n",
" collect += ' energy'\n",
"\n",
"# Run the benchmark in all the configured governors\n",
"for conf_name in test_confs:\n",
"\n",
" for idx,wload_name in enumerate(test_wloads):\n",
" \n",
" wload_kind = wload_name.split()[0]\n",
" logging.info('------------------------')\n",
" logging.info('Test %d/%d: %s in %s configuration',\n",
" idx+1, len(test_wloads), wload_kind.upper(), conf_name.upper())\n",
" res = experiment(wl, te.res_dir, conf_name, wload_name, iterations, collect)\n",
" results[conf_name][wload_name] = res\n",
"\n",
" # Save collected results\n",
" conf_dir = os.path.join(te.res_dir, conf_name)\n",
" if not os.path.isdir(conf_dir): os.mkdir(conf_dir)\n",
" res_file = os.path.join(conf_dir, 'results.json')\n",
" with open(res_file, 'w') as fh:\n",
" json.dump(results[conf_name], fh, indent=4)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Energy Measurements Report"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Energy consumption STD, YOUTUBE HTTPS://YOUTU.BE/XSGBVZEBUBK?T=45S 15 : NaN\n",
"Energy consumption STD, JANKBENCH LIST_VIEW : NaN\n"
]
}
],
"source": [
"for conf_name in test_confs:\n",
" for idx,wload_name in enumerate(test_wloads):\n",
" nrg = 'NaN'\n",
" result = results[conf_name][wload_name]\n",
" if 'nrg_report' in result and result['nrg_report']:\n",
" nrg = '{:6.1f}'.format(float(result['nrg_report'].channels['BAT']))\n",
" print \"Energy consumption {}, {:52}: {}\".format(conf_name.upper(), wload_name.upper(), nrg)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
},
"toc": {
"toc_cell": false,
"toc_number_sections": true,
"toc_threshold": 6,
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 0
}