| { |
| "cells": [ |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "# EAS Testing - YouTube on Android" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "The goal of this experiment is to run Youtube videos on a Nexus N5X running Android with an EAS kernel and collect results. The Analysis phase will consist in comparing EAS with other schedulers, that is comparing *sched* governor with:\n", |
| "\n", |
| " - interactive\n", |
| " - performance\n", |
| " - powersave\n", |
| " - ondemand" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 1, |
| "metadata": { |
| "collapsed": false |
| }, |
| "outputs": [ |
| { |
| "name": "stderr", |
| "output_type": "stream", |
| "text": [ |
| "2016-12-06 19:46:08,105 INFO : root : Using LISA logging configuration:\n", |
| "2016-12-06 19:46:08,106 INFO : root : /home/vagrant/lisa/logging.conf\n" |
| ] |
| } |
| ], |
| "source": [ |
| "import logging\n", |
| "from conf import LisaLogging\n", |
| "LisaLogging.setup()" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 2, |
| "metadata": { |
| "collapsed": false |
| }, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "Populating the interactive namespace from numpy and matplotlib\n" |
| ] |
| } |
| ], |
| "source": [ |
| "%pylab inline\n", |
| "\n", |
| "import os\n", |
| "from time import sleep\n", |
| "\n", |
| "# Support to access the remote target\n", |
| "import devlib\n", |
| "from env import TestEnv\n", |
| "\n", |
| "from devlib.utils.android import adb_command\n", |
| "\n", |
| "# Import support for Android devices\n", |
| "from android import System\n", |
| "\n", |
| "# Support for trace events analysis\n", |
| "from trace import Trace\n", |
| "\n", |
| "# Suport for FTrace events parsing and visualization\n", |
| "import trappy" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "# Test Environment set up" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "In case more than one Android device are conencted to the host, you must specify the ID of the device you want to target in `my_target_conf`. Run `adb devices` on your host to get the ID." |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 3, |
| "metadata": { |
| "collapsed": false |
| }, |
| "outputs": [], |
| "source": [ |
| "# Setup a target configuration\n", |
| "my_target_conf = {\n", |
| " \n", |
| " # Target platform and board\n", |
| " \"platform\" : 'android',\n", |
| "\n", |
| " # Add target support\n", |
| " \"board\" : 'pixel',\n", |
| " \n", |
| " # Device ID\n", |
| " \"device\" : \"HT6670300102\",\n", |
| " \n", |
| " # ANDROID_HOME\n", |
| " \"ANDROID_HOME\" : '/home/vagrant/lisa/tools/android-sdk-linux/',\n", |
| " \n", |
| " # Define devlib modules to load\n", |
| " \"modules\" : [\n", |
| " 'cpufreq' # enable CPUFreq support\n", |
| " ],\n", |
| "}" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 4, |
| "metadata": { |
| "collapsed": false, |
| "scrolled": false |
| }, |
| "outputs": [], |
| "source": [ |
| "my_tests_conf = {\n", |
| "\n", |
| " # Folder where all the results will be collected\n", |
| " \"results_dir\" : \"Android_Youtube\",\n", |
| "\n", |
| " # Platform configurations to test\n", |
| " \"confs\" : [\n", |
| " {\n", |
| " \"tag\" : \"youtube\",\n", |
| " \"flags\" : \"ftrace\", # Enable FTrace events\n", |
| " \"sched_features\" : \"ENERGY_AWARE\", # enable EAS\n", |
| " },\n", |
| " ],\n", |
| " \n", |
| " # FTrace events to collect for all the tests configuration which have\n", |
| " # the \"ftrace\" flag enabled\n", |
| " \"ftrace\" : {\n", |
| " \"events\" : [\n", |
| " \"sched_switch\",\n", |
| " \"sched_load_avg_cpu\",\n", |
| " \"cpu_frequency\",\n", |
| " \"cpu_capacity\"\n", |
| " ],\n", |
| " \"buffsize\" : 10 * 1024,\n", |
| " },\n", |
| " \n", |
| " # Tools required by the experiments\n", |
| " \"tools\" : [ 'trace-cmd' ],\n", |
| "}" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 5, |
| "metadata": { |
| "collapsed": false |
| }, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "adbd is already running as root\r\n" |
| ] |
| } |
| ], |
| "source": [ |
| "# Ensure ADB has root priviledges, which are required by systrace\n", |
| "!adb root" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 6, |
| "metadata": { |
| "collapsed": false, |
| "scrolled": true |
| }, |
| "outputs": [ |
| { |
| "name": "stderr", |
| "output_type": "stream", |
| "text": [ |
| "2016-12-06 19:46:11,262 INFO : TestEnv : Using base path: /home/vagrant/lisa\n", |
| "2016-12-06 19:46:11,263 INFO : TestEnv : Loading custom (inline) target configuration\n", |
| "2016-12-06 19:46:11,264 INFO : TestEnv : Loading custom (inline) test configuration\n", |
| "2016-12-06 19:46:11,264 INFO : TestEnv : External tools using:\n", |
| "2016-12-06 19:46:11,264 INFO : TestEnv : ANDROID_HOME: /home/vagrant/lisa/tools/android-sdk-linux/\n", |
| "2016-12-06 19:46:11,265 INFO : TestEnv : CATAPULT_HOME: /home/vagrant/lisa/tools/catapult\n", |
| "2016-12-06 19:46:11,265 INFO : TestEnv : Loading board:\n", |
| "2016-12-06 19:46:11,266 INFO : TestEnv : /home/vagrant/lisa/libs/utils/platforms/pixel.json\n", |
| "2016-12-06 19:46:11,266 INFO : TestEnv : Devlib modules to load: [u'bl', u'cpufreq']\n", |
| "2016-12-06 19:46:11,267 INFO : TestEnv : Connecting Android target [HT6670300102]\n", |
| "2016-12-06 19:46:11,267 INFO : TestEnv : Connection settings:\n", |
| "2016-12-06 19:46:11,267 INFO : TestEnv : {'device': 'HT6670300102'}\n", |
| "2016-12-06 19:46:11,355 INFO : android : ls command is set to ls -1\n", |
| "2016-12-06 19:46:12,105 INFO : TestEnv : Initializing target workdir:\n", |
| "2016-12-06 19:46:12,108 INFO : TestEnv : /data/local/tmp/devlib-target\n", |
| "2016-12-06 19:46:14,169 INFO : TestEnv : Topology:\n", |
| "2016-12-06 19:46:14,171 INFO : TestEnv : [[0, 1], [2, 3]]\n", |
| "2016-12-06 19:46:14,400 INFO : TestEnv : Loading default EM:\n", |
| "2016-12-06 19:46:14,400 INFO : TestEnv : /home/vagrant/lisa/libs/utils/platforms/pixel.json\n", |
| "2016-12-06 19:46:15,115 INFO : TestEnv : Enabled tracepoints:\n", |
| "2016-12-06 19:46:15,116 INFO : TestEnv : sched_switch\n", |
| "2016-12-06 19:46:15,116 INFO : TestEnv : sched_load_avg_cpu\n", |
| "2016-12-06 19:46:15,117 INFO : TestEnv : cpu_frequency\n", |
| "2016-12-06 19:46:15,117 INFO : TestEnv : cpu_capacity\n", |
| "2016-12-06 19:46:15,118 WARNING : TestEnv : Wipe previous contents of the results folder:\n", |
| "2016-12-06 19:46:15,118 WARNING : TestEnv : /home/vagrant/lisa/results/Android_Youtube\n", |
| "2016-12-06 19:46:15,129 INFO : TestEnv : Set results folder to:\n", |
| "2016-12-06 19:46:15,129 INFO : TestEnv : /home/vagrant/lisa/results/Android_Youtube\n", |
| "2016-12-06 19:46:15,130 INFO : TestEnv : Experiment results available also in:\n", |
| "2016-12-06 19:46:15,130 INFO : TestEnv : /home/vagrant/lisa/results_latest\n" |
| ] |
| } |
| ], |
| "source": [ |
| "# Initialize a test environment using:\n", |
| "# the provided target configuration (my_target_conf)\n", |
| "# the provided test configuration (my_test_conf)\n", |
| "te = TestEnv(target_conf=my_target_conf, test_conf=my_tests_conf)\n", |
| "target = te.target" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "# Support Functions" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "This set of support functions will help us running the benchmark using different CPUFreq governors." |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 7, |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [ |
| "def set_performance():\n", |
| " target.cpufreq.set_all_governors('performance')\n", |
| "\n", |
| "def set_powersave():\n", |
| " target.cpufreq.set_all_governors('powersave')\n", |
| "\n", |
| "def set_interactive():\n", |
| " target.cpufreq.set_all_governors('interactive')\n", |
| "\n", |
| "def set_sched():\n", |
| " target.cpufreq.set_all_governors('sched')\n", |
| "\n", |
| "def set_ondemand():\n", |
| " target.cpufreq.set_all_governors('ondemand')\n", |
| " \n", |
| " for cpu in target.list_online_cpus():\n", |
| " tunables = target.cpufreq.get_governor_tunables(cpu)\n", |
| " target.cpufreq.set_governor_tunables(\n", |
| " cpu,\n", |
| " 'ondemand',\n", |
| " **{'sampling_rate' : tunables['sampling_rate_min']}\n", |
| " )" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 8, |
| "metadata": { |
| "collapsed": false |
| }, |
| "outputs": [], |
| "source": [ |
| "# CPUFreq configurations to test\n", |
| "confs = {\n", |
| " 'performance' : {\n", |
| " 'label' : 'prf',\n", |
| " 'set' : set_performance,\n", |
| " },\n", |
| " #'powersave' : {\n", |
| " # 'label' : 'pws',\n", |
| " # 'set' : set_powersave,\n", |
| " #},\n", |
| " 'interactive' : {\n", |
| " 'label' : 'int',\n", |
| " 'set' : set_interactive,\n", |
| " },\n", |
| " #'sched' : {\n", |
| " # 'label' : 'sch',\n", |
| " # 'set' : set_sched,\n", |
| " #},\n", |
| " #'ondemand' : {\n", |
| " # 'label' : 'odm',\n", |
| " # 'set' : set_ondemand,\n", |
| " #}\n", |
| "}\n", |
| "\n", |
| "# The set of results for each comparison test\n", |
| "results = {}" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 9, |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [ |
| "YOUTUBE_CMD = 'shell dumpsys gfxinfo com.google.android.youtube > {}'\n", |
| "\n", |
| "def youtube_run(exp_dir, video_url, video_duration_s):\n", |
| " # Unlock device screen (assume no password required)\n", |
| " target.execute('input keyevent 82')\n", |
| " # Press Back button to be sure we run the video from the start\n", |
| " target.execute('input keyevent KEYCODE_BACK')\n", |
| "\n", |
| " # Start YouTube video on the target device\n", |
| " target.execute('am start -a android.intent.action.VIEW \"{}\"'.format(video_url))\n", |
| " # Allow the activity to start\n", |
| " sleep(3)\n", |
| " # Reset framestats collection\n", |
| " target.execute('dumpsys gfxinfo --reset')\n", |
| " # Wait until the end of the video\n", |
| " sleep(video_duration_s)\n", |
| " \n", |
| " # Get frame stats\n", |
| " framestats_file = os.path.join(exp_dir, \"framestats.txt\")\n", |
| " adb_command(target.adb_name, YOUTUBE_CMD.format(framestats_file))\n", |
| "\n", |
| " # Close application\n", |
| " target.execute('am force-stop com.google.android.youtube')\n", |
| "\n", |
| " # Clear application data\n", |
| " target.execute('pm clear com.google.android.youtube')\n", |
| "\n", |
| " return framestats_file" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 10, |
| "metadata": { |
| "collapsed": false |
| }, |
| "outputs": [], |
| "source": [ |
| "def experiment(governor, exp_dir, collect='ftrace', trace_time=30):\n", |
| " os.system('mkdir -p {}'.format(exp_dir));\n", |
| "\n", |
| " logging.info('------------------------')\n", |
| " logging.info('Run workload using %s governor', governor)\n", |
| " confs[governor]['set']()\n", |
| "\n", |
| " # Start the required tracing command\n", |
| " if 'ftrace' in collect:\n", |
| " # Start FTrace and Energy monitoring\n", |
| " te.ftrace.start()\n", |
| " elif 'systrace' in collect:\n", |
| " # Start systrace\n", |
| " trace_file = os.path.join(exp_dir, 'trace.html')\n", |
| " systrace_output = System.systrace_start(te, trace_file, trace_time)\n", |
| "\n", |
| " ### Run the benchmark ###\n", |
| " framestats_file = youtube_run(exp_dir, \"https://youtu.be/XSGBVzeBUbk?t=45s\", trace_time)\n", |
| "\n", |
| " # Stop the required trace command\n", |
| " if 'ftrace' in collect:\n", |
| " te.ftrace.stop()\n", |
| " # Collect and keep track of the trace\n", |
| " trace_file = os.path.join(exp_dir, 'trace.dat')\n", |
| " te.ftrace.get_trace(trace_file)\n", |
| " elif 'systrace' in collect:\n", |
| " if systrace_output:\n", |
| " logging.info('Waiting systrace report [%s]...', trace_file)\n", |
| " systrace_output.wait()\n", |
| " else:\n", |
| " logging.warning('Systrace is not running!') \n", |
| "\n", |
| " # Parse trace\n", |
| " tr = Trace(te.platform, trace_file,\n", |
| " events=my_tests_conf['ftrace']['events'])\n", |
| "\n", |
| " # return all the experiment data\n", |
| " return {\n", |
| " 'dir' : exp_dir,\n", |
| " 'framestats_file' : framestats_file,\n", |
| " 'trace' : trace_file,\n", |
| " 'ftrace' : tr.ftrace\n", |
| " }" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "# Run experiments and collect traces" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 11, |
| "metadata": { |
| "collapsed": false, |
| "scrolled": true |
| }, |
| "outputs": [ |
| { |
| "name": "stderr", |
| "output_type": "stream", |
| "text": [ |
| "2016-12-06 19:46:21,457 INFO : root : ------------------------\n", |
| "2016-12-06 19:46:21,458 INFO : root : Run workload using performance governor\n", |
| "2016-12-06 19:46:21,556 INFO : System : SysTrace: /home/vagrant/lisa/tools/catapult/systrace/systrace/run_systrace.py -e HT6670300102 -o /home/vagrant/lisa/results/Android_Youtube/performance/trace.html gfx view sched freq idle -t 15\n", |
| "2016-12-06 19:46:42,226 INFO : root : Waiting systrace report [/home/vagrant/lisa/results/Android_Youtube/performance/trace.html]...\n", |
| "2016-12-06 19:46:43,203 INFO : Trace : Parsing SysTrace format...\n", |
| "2016-12-06 19:46:48,092 INFO : Trace : Collected events spans a 9.888 [s] time interval\n", |
| "2016-12-06 19:46:48,092 INFO : Trace : Set plots time range to (0.000000, 9.888454)[s]\n", |
| "2016-12-06 19:46:48,093 INFO : Analysis : Registering trace analysis modules:\n", |
| "2016-12-06 19:46:48,094 INFO : Analysis : tasks\n", |
| "2016-12-06 19:46:48,094 INFO : Analysis : status\n", |
| "2016-12-06 19:46:48,095 INFO : Analysis : frequency\n", |
| "2016-12-06 19:46:48,096 INFO : Analysis : cpus\n", |
| "2016-12-06 19:46:48,097 INFO : Analysis : latency\n", |
| "2016-12-06 19:46:48,097 INFO : Analysis : idle\n", |
| "2016-12-06 19:46:48,098 INFO : Analysis : functions\n", |
| "2016-12-06 19:46:48,098 INFO : Analysis : eas\n", |
| "2016-12-06 19:46:48,208 INFO : root : ------------------------\n", |
| "2016-12-06 19:46:48,209 INFO : root : Run workload using interactive governor\n", |
| "2016-12-06 19:46:48,316 INFO : System : SysTrace: /home/vagrant/lisa/tools/catapult/systrace/systrace/run_systrace.py -e HT6670300102 -o /home/vagrant/lisa/results/Android_Youtube/interactive/trace.html gfx view sched freq idle -t 15\n", |
| "2016-12-06 19:47:09,175 INFO : root : Waiting systrace report [/home/vagrant/lisa/results/Android_Youtube/interactive/trace.html]...\n", |
| "2016-12-06 19:47:10,185 INFO : Trace : Parsing SysTrace format...\n", |
| "2016-12-06 19:47:13,573 INFO : Trace : Platform clusters verified to be Frequency coherent\n", |
| "2016-12-06 19:47:15,675 INFO : Trace : Collected events spans a 8.481 [s] time interval\n", |
| "2016-12-06 19:47:15,675 INFO : Trace : Set plots time range to (0.000000, 8.480988)[s]\n", |
| "2016-12-06 19:47:15,676 INFO : Analysis : Registering trace analysis modules:\n", |
| "2016-12-06 19:47:15,676 INFO : Analysis : tasks\n", |
| "2016-12-06 19:47:15,677 INFO : Analysis : status\n", |
| "2016-12-06 19:47:15,677 INFO : Analysis : frequency\n", |
| "2016-12-06 19:47:15,678 INFO : Analysis : cpus\n", |
| "2016-12-06 19:47:15,678 INFO : Analysis : latency\n", |
| "2016-12-06 19:47:15,679 INFO : Analysis : idle\n", |
| "2016-12-06 19:47:15,680 INFO : Analysis : functions\n", |
| "2016-12-06 19:47:15,680 INFO : Analysis : eas\n" |
| ] |
| } |
| ], |
| "source": [ |
| "# Run the benchmark in all the configured governors\n", |
| "for governor in confs:\n", |
| " test_dir = os.path.join(te.res_dir, governor)\n", |
| " results[governor] = experiment(governor, test_dir,\n", |
| " collect='systrace', trace_time=15)" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": { |
| "collapsed": false |
| }, |
| "source": [ |
| "# UI Performance Analysis" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 12, |
| "metadata": { |
| "collapsed": false |
| }, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "Frame Statistics for PERFORMANCE governor\n", |
| "Stats since: 107266916813060ns\n", |
| "Total frames rendered: 747\n", |
| "Janky frames: 44 (5.89%)\n", |
| "50th percentile: 5ms\n", |
| "90th percentile: 8ms\n", |
| "95th percentile: 19ms\n", |
| "99th percentile: 113ms\n", |
| "\n", |
| "Frame Statistics for INTERACTIVE governor\n", |
| "Stats since: 107266916813060ns\n", |
| "Total frames rendered: 942\n", |
| "Janky frames: 60 (6.37%)\n", |
| "50th percentile: 5ms\n", |
| "90th percentile: 9ms\n", |
| "95th percentile: 20ms\n", |
| "99th percentile: 113ms\n", |
| "\n" |
| ] |
| } |
| ], |
| "source": [ |
| "for governor in confs:\n", |
| " framestats_file = results[governor]['framestats_file']\n", |
| " print \"Frame Statistics for {} governor\".format(governor.upper())\n", |
| " !sed '/Stats since/,/99th/!d;/99th/q' $framestats_file\n", |
| " print \"\"" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 13, |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [ |
| "trace_file = results['interactive']['trace']\n", |
| "!xdg-open {trace_file}" |
| ] |
| } |
| ], |
| "metadata": { |
| "kernelspec": { |
| "display_name": "Python 2", |
| "language": "python", |
| "name": "python2" |
| }, |
| "language_info": { |
| "codemirror_mode": { |
| "name": "ipython", |
| "version": 2 |
| }, |
| "file_extension": ".py", |
| "mimetype": "text/x-python", |
| "name": "python", |
| "nbconvert_exporter": "python", |
| "pygments_lexer": "ipython2", |
| "version": "2.7.6" |
| }, |
| "toc": { |
| "toc_cell": false, |
| "toc_number_sections": true, |
| "toc_threshold": 6, |
| "toc_window_display": false |
| } |
| }, |
| "nbformat": 4, |
| "nbformat_minor": 0 |
| } |