ipynb: executor: add example usage of the executor module
The libs/utils/executor.py module provides a simple yet useful execution
engine to run a reconfigured set of experiments.
This patch adds an example Notebook which shows how to configure and
use the execution engine.
Comments in the notebook should provide enough information to understand
how to use that engine from within a Notebook to run a set of experiments.
Signed-off-by: Patrick Bellasi <patrick.bellasi@arm.com>
diff --git a/README.md b/README.md
index dca0bbc..ca9f60f 100644
--- a/README.md
+++ b/README.md
@@ -157,7 +157,7 @@
3. Run the EAS RFC test using the standard nosetest command:
```sh
- $ nosetests -v tests/eas/rfc.py:EAS
+ $ nosetests -v tests/eas/rfc.py
```
4. Wait for the test to complete and than you can report the results with:
@@ -278,7 +278,23 @@
the TRAPpy library
7. visualize some simple performance metrics for the tasks
-## 4. Regression test example 1
+## 4. Get confident with the tests execution engine
+
+Tests are usually performed on a set of data collected while executing properly
+defined experiments. An experiment is usually defined by a specific **target
+configuration** which is used to run a certain **workload mix**.
+Thus, to run experiments we usually need the support of a proper module which
+configure a target and execute a workload on it in order to collect the data
+required for a test.
+
+The __Executor__ module is a simple yet effective support to collect all the
+data required for a test. This notebook:
+[utils/executor_example.ipynb](http://localhost:8888/notebooks/utils/executor_example.ipynb)
+is a simple example of how to use the Executor collect experimental data for a
+predefined set of target configurations and by running a specified set of
+workloads.
+
+## 5. Regression test example 1
One of the main aims of LISA is to become a repository for regression
tests on scheduler and power management behavior. A common pattern for
@@ -313,7 +329,7 @@
experiments which can then be transformed into a standalone regression
test.
-## 5. Regression test example 2
+## 6. Regression test example 2
Once a new set of tests have been defined and verified, perhaps by using
a notebook to develop them, they can be transformed into a standalone
diff --git a/ipynb/utils/executor_example.ipynb b/ipynb/utils/executor_example.ipynb
new file mode 100644
index 0000000..26dfe7e
--- /dev/null
+++ b/ipynb/utils/executor_example.ipynb
@@ -0,0 +1,410 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "import logging\n",
+ "reload(logging)\n",
+ "log_fmt = '%(asctime)-9s %(levelname)-8s: %(message)s'\n",
+ "logging.basicConfig(format=log_fmt)\n",
+ "\n",
+ "# Change to info once the notebook runs ok\n",
+ "#logging.getLogger().setLevel(logging.DEBUG)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Populating the interactive namespace from numpy and matplotlib\n"
+ ]
+ }
+ ],
+ "source": [
+ "%pylab inline\n",
+ "\n",
+ "import datetime\n",
+ "import devlib\n",
+ "import os\n",
+ "import json\n",
+ "import pandas as pd\n",
+ "import re\n",
+ "import subprocess\n",
+ "import trappy\n",
+ "from trappy.plotter.Utils import get_trace_event_data\n",
+ "\n",
+ "import matplotlib.gridspec as gridspec\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "# Support to access the remote target\n",
+ "import devlib\n",
+ "from env import TestEnv\n",
+ "\n",
+ "from executor import Executor"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Target Configuration"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "# Setup a target configuration\n",
+ "my_target_conf = {\n",
+ " \n",
+ " # Target platform and board\n",
+ " \"platform\" : 'linux',\n",
+ " \"board\" : 'aboard',\n",
+ " \n",
+ " # Target board IP/MAC address\n",
+ " \"host\" : '192.168.0.1',\n",
+ " \n",
+ " # Login credentials\n",
+ " \"username\" : 'root',\n",
+ " \"password\" : 'test0000',\n",
+ "\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Tests Configuration"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "collapsed": false,
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "my_tests_conf = {\n",
+ "\n",
+ " # This is the output folder where all the results will be collected\n",
+ " \"id\" : \"ChromeOS_Profiling\",\n",
+ "\n",
+ " # Platform configurations to test\n",
+ " \"confs\" : [\n",
+ " {\n",
+ " \"tag\" : \"base\",\n",
+ " \"flags\" : \"ftrace\",\n",
+ " \"sched_features\" : \"NO_ENERGY_AWARE\",\n",
+ " \"cpufreq\" : {\n",
+ " \"governor\" : \"performance\",\n",
+ " },\n",
+ " },\n",
+ " {\n",
+ " \"tag\" : \"eas\",\n",
+ " \"flags\" : \"ftrace\",\n",
+ " \"sched_features\" : \"ENERGY_AWARE\",\n",
+ " \"cpufreq\" : {\n",
+ " \"governor\" : \"performance\",\n",
+ " },\n",
+ " },\n",
+ " ],\n",
+ " \n",
+ " # Workloads to run (on each platform configuration)\n",
+ " \"wloads\" : {\n",
+ " \"perf\" : {\n",
+ " \"type\" : \"perf_bench\",\n",
+ " \"conf\" : {\n",
+ " \"class\" : \"messaging\",\n",
+ " \"params\" : {\n",
+ " \"group\" : 1,\n",
+ " \"loop\" : 10,\n",
+ " \"pipe\" : True,\n",
+ " \"thread\": True,\n",
+ " }\n",
+ " }\n",
+ " },\n",
+ " \"rta\" : {\n",
+ " \"type\" : \"rt-app\",\n",
+ " \"loadref\" : \"big\",\n",
+ " \"conf\" : {\n",
+ " \"class\" : \"profile\",\n",
+ " \"params\" : {\n",
+ " \"p20\" : {\n",
+ " \"kind\" : \"periodic\",\n",
+ " \"params\" : {\n",
+ " \"duty_cycle_pct\" : 20,\n",
+ " },\n",
+ " },\n",
+ " },\n",
+ " },\n",
+ " },\n",
+ " },\n",
+ " \n",
+ " # Number of iterations for each workload\n",
+ " \"iterations\" : 1,\n",
+ " \n",
+ " # FTrace events to collect and functions to profile for all the\n",
+ " # tests configuration which have the \"ftrace\" flag enabled\n",
+ " \"ftrace\" : {\n",
+ " \"events\" : [\n",
+ " \"sched_switch\",\n",
+ " \"sched_wakeup\",\n",
+ " \"sched_wakeup_new\",\n",
+ " \"cpu_frequency\",\n",
+ " ],\n",
+ " \"buffsize\" : 80 * 1024,\n",
+ " },\n",
+ " \n",
+ " # Tools required by the experiments\n",
+ " \"tools\" : [ 'trace-cmd', 'perf' ],\n",
+ " \n",
+ " # Modules required by these experiments\n",
+ " \"modules\" : [ 'bl', 'cpufreq' ],\n",
+ "\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Tests execution"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "04:04:11 INFO : Target - Loading custom (inline) test configuration\n",
+ "04:04:11 INFO : Target - Using base path: /home/derkling/Code/schedtest\n",
+ "04:04:11 INFO : Target - Loading custom (inline) target configuration\n",
+ "04:04:11 INFO : Target - Loading custom (inline) test configuration\n",
+ "04:04:11 INFO : Target - Devlib modules to load: ['bl', 'cpufreq']\n",
+ "04:04:11 INFO : Target - Connecting linux target with: {'username': 'root', 'host': '192.168.0.1', 'password': 'test0000'}\n",
+ "04:04:16 INFO : Target - Initializing target workdir [/root/devlib-target]\n",
+ "04:04:20 INFO : Target topology: [[0, 1], [2, 3]]\n",
+ "04:04:23 INFO : Platform - Loading default EM [/home/derkling/Code/schedtest/libs/utils/platforms/aboard.json]...\n",
+ "04:04:24 INFO : FTrace - Enabled events:\n",
+ "04:04:24 INFO : FTrace - ['sched_switch', 'sched_wakeup', 'sched_wakeup_new', 'cpu_frequency']\n",
+ "04:04:24 INFO : FTrace - None\n",
+ "04:04:24 INFO : TestEnv - Set results folder to:\n",
+ "04:04:24 INFO : TestEnv - /home/derkling/Code/schedtest/results/ChromeOS_Profiling\n",
+ "04:04:24 INFO : TestEnv - Experiment results available also in:\n",
+ "04:04:24 INFO : TestEnv - /home/derkling/Code/schedtest/results_latest\n",
+ "04:04:24 INFO : \n",
+ "04:04:24 INFO : ################################################################################\n",
+ "04:04:24 INFO : Executor - Experiments configuration\n",
+ "04:04:24 INFO : ################################################################################\n",
+ "04:04:24 INFO : Executor - Configured to run:\n",
+ "04:04:24 INFO : Executor - 2 targt configurations:\n",
+ "04:04:24 INFO : Executor - base, eas\n",
+ "04:04:24 INFO : Executor - 2 workloads (1 iterations each)\n",
+ "04:04:24 INFO : Executor - rta, perf\n",
+ "04:04:24 INFO : Executor - Total: 4 experiments\n",
+ "04:04:24 INFO : Executor - Results will be collected under:\n",
+ "04:04:24 INFO : Executor - /home/derkling/Code/schedtest/results/ChromeOS_Profiling\n"
+ ]
+ }
+ ],
+ "source": [
+ "executor = Executor(my_target_conf, my_tests_conf)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "04:04:24 INFO : \n",
+ "04:04:24 INFO : ################################################################################\n",
+ "04:04:24 INFO : Executor - Experiments execution\n",
+ "04:04:24 INFO : ################################################################################\n",
+ "04:04:24 INFO : \n",
+ "04:04:24 INFO : ================================================================================\n",
+ "04:04:24 INFO : TargetConfig - configuring target for [base] experiments\n",
+ "04:04:26 INFO : CPUFreq - Configuring all CPUs to use [performance] governor\n",
+ "04:04:27 INFO : WlGen - Setup new workload rta\n",
+ "04:04:27 INFO : RTApp - Workload duration defined by longest task\n",
+ "04:04:27 INFO : RTApp - Default policy: SCHED_OTHER\n",
+ "04:04:27 INFO : RTApp - ------------------------\n",
+ "04:04:27 INFO : RTApp - task [task_p20], sched: using default policy\n",
+ "04:04:27 INFO : RTApp - | calibration CPU: 2\n",
+ "04:04:27 INFO : RTApp - | loops count: 1\n",
+ "04:04:27 INFO : RTApp - + phase_000001: duration 1.000000 [s] (10 loops)\n",
+ "04:04:27 INFO : RTApp - | period 100000 [us], duty_cycle 20 %\n",
+ "04:04:27 INFO : RTApp - | run_time 20000 [us], sleep_time 80000 [us]\n",
+ "04:04:28 INFO : ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
+ "04:04:28 INFO : Executor - Experiment 1/4, [base:rta] 1/1\n",
+ "04:04:28 WARNING : Executor - FTrace events collection enabled\n",
+ "04:04:34 INFO : WlGen - WlGen [start]: /root/devlib-target/bin/rt-app /root/devlib-target/run_dir/rta_00.json\n",
+ "04:04:40 INFO : WlGen - Setup new workload perf\n",
+ "04:04:41 INFO : ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
+ "04:04:41 INFO : Executor - Experiment 2/4, [base:perf] 1/1\n",
+ "04:04:41 WARNING : Executor - FTrace events collection enabled\n",
+ "04:04:47 INFO : WlGen - WlGen [start]: /root/devlib-target/bin/perf bench sched messaging --pipe --thread --group 1 --loop 10\n",
+ "04:04:47 INFO : PerfBench - Completion time: 0.016000, Performance 62.500000\n",
+ "04:04:52 INFO : \n",
+ "04:04:52 INFO : ================================================================================\n",
+ "04:04:52 INFO : TargetConfig - configuring target for [eas] experiments\n",
+ "04:04:54 INFO : CPUFreq - Configuring all CPUs to use [performance] governor\n",
+ "04:04:54 INFO : WlGen - Setup new workload rta\n",
+ "04:04:54 INFO : RTApp - Workload duration defined by longest task\n",
+ "04:04:54 INFO : RTApp - Default policy: SCHED_OTHER\n",
+ "04:04:54 INFO : RTApp - ------------------------\n",
+ "04:04:54 INFO : RTApp - task [task_p20], sched: using default policy\n",
+ "04:04:54 INFO : RTApp - | calibration CPU: 2\n",
+ "04:04:54 INFO : RTApp - | loops count: 1\n",
+ "04:04:54 INFO : RTApp - + phase_000001: duration 1.000000 [s] (10 loops)\n",
+ "04:04:54 INFO : RTApp - | period 100000 [us], duty_cycle 20 %\n",
+ "04:04:54 INFO : RTApp - | run_time 20000 [us], sleep_time 80000 [us]\n",
+ "04:04:55 INFO : ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
+ "04:04:55 INFO : Executor - Experiment 3/4, [eas:rta] 1/1\n",
+ "04:04:55 WARNING : Executor - FTrace events collection enabled\n",
+ "04:05:00 INFO : WlGen - WlGen [start]: /root/devlib-target/bin/rt-app /root/devlib-target/run_dir/rta_00.json\n",
+ "04:05:06 INFO : WlGen - Setup new workload perf\n",
+ "04:05:06 INFO : ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
+ "04:05:06 INFO : Executor - Experiment 4/4, [eas:perf] 1/1\n",
+ "04:05:06 WARNING : Executor - FTrace events collection enabled\n",
+ "04:05:12 INFO : WlGen - WlGen [start]: /root/devlib-target/bin/perf bench sched messaging --pipe --thread --group 1 --loop 10\n",
+ "04:05:12 INFO : PerfBench - Completion time: 0.021000, Performance 47.619048\n",
+ "04:05:17 INFO : \n",
+ "04:05:17 INFO : ################################################################################\n",
+ "04:05:17 INFO : Executor - Experiments execution completed\n",
+ "04:05:17 INFO : ################################################################################\n",
+ "04:05:17 INFO : Executor - Results available in:\n",
+ "04:05:17 INFO : Executor - /home/derkling/Code/schedtest/results/ChromeOS_Profiling\n"
+ ]
+ }
+ ],
+ "source": [
+ "executor.run()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u001b[01;34m/home/derkling/Code/schedtest/results/ChromeOS_Profiling\u001b[00m\r\n",
+ "├── \u001b[01;34mperf_bench_messaging:base:perf\u001b[00m\r\n",
+ "│ ├── \u001b[01;34m1\u001b[00m\r\n",
+ "│ │ ├── output.log\r\n",
+ "│ │ ├── performance.json\r\n",
+ "│ │ └── trace.dat\r\n",
+ "│ ├── kernel.config\r\n",
+ "│ ├── kernel.version\r\n",
+ "│ └── platform.json\r\n",
+ "├── \u001b[01;34mperf_bench_messaging:eas:perf\u001b[00m\r\n",
+ "│ ├── \u001b[01;34m1\u001b[00m\r\n",
+ "│ │ ├── output.log\r\n",
+ "│ │ ├── performance.json\r\n",
+ "│ │ └── trace.dat\r\n",
+ "│ ├── kernel.config\r\n",
+ "│ ├── kernel.version\r\n",
+ "│ └── platform.json\r\n",
+ "├── \u001b[01;34mrtapp:base:rta\u001b[00m\r\n",
+ "│ ├── \u001b[01;34m1\u001b[00m\r\n",
+ "│ │ ├── output.log\r\n",
+ "│ │ ├── rta_00.json\r\n",
+ "│ │ ├── rt-app-task_p20-0.log\r\n",
+ "│ │ └── trace.dat\r\n",
+ "│ ├── kernel.config\r\n",
+ "│ ├── kernel.version\r\n",
+ "│ └── platform.json\r\n",
+ "├── \u001b[01;34mrtapp:base:single\u001b[00m\r\n",
+ "│ ├── \u001b[01;34m1\u001b[00m\r\n",
+ "│ │ ├── output.log\r\n",
+ "│ │ ├── rt-app-task_p20-0.log\r\n",
+ "│ │ ├── single_00.json\r\n",
+ "│ │ └── trace.dat\r\n",
+ "│ ├── kernel.config\r\n",
+ "│ ├── kernel.version\r\n",
+ "│ └── platform.json\r\n",
+ "├── \u001b[01;34mrtapp:eas:rta\u001b[00m\r\n",
+ "│ ├── \u001b[01;34m1\u001b[00m\r\n",
+ "│ │ ├── output.log\r\n",
+ "│ │ ├── rta_00.json\r\n",
+ "│ │ ├── rt-app-task_p20-0.log\r\n",
+ "│ │ └── trace.dat\r\n",
+ "│ ├── kernel.config\r\n",
+ "│ ├── kernel.version\r\n",
+ "│ └── platform.json\r\n",
+ "└── \u001b[01;34mrtapp:eas:single\u001b[00m\r\n",
+ " ├── \u001b[01;34m1\u001b[00m\r\n",
+ " │ ├── output.log\r\n",
+ " │ ├── rt-app-task_p20-0.log\r\n",
+ " │ ├── single_00.json\r\n",
+ " │ └── trace.dat\r\n",
+ " ├── kernel.config\r\n",
+ " ├── kernel.version\r\n",
+ " └── platform.json\r\n",
+ "\r\n",
+ "12 directories, 40 files\r\n"
+ ]
+ }
+ ],
+ "source": [
+ "!tree {executor.te.res_dir}"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.9"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}