Merge pull request #163 from marcbonnici/Derived_Measurements
Add support for AcmeCape and Derived Measurements
diff --git a/devlib/__init__.py b/devlib/__init__.py
index 2f50632..b1b4fa3 100644
--- a/devlib/__init__.py
+++ b/devlib/__init__.py
@@ -19,6 +19,9 @@
from devlib.instrument.netstats import NetstatsInstrument
from devlib.instrument.gem5power import Gem5PowerInstrument
+from devlib.derived import DerivedMeasurements
+from devlib.derived.derived_measurements import DerivedEnergyMeasurements
+
from devlib.trace.ftrace import FtraceCollector
from devlib.host import LocalConnection
diff --git a/devlib/derived/__init__.py b/devlib/derived/__init__.py
new file mode 100644
index 0000000..5689a58
--- /dev/null
+++ b/devlib/derived/__init__.py
@@ -0,0 +1,19 @@
+# Copyright 2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+class DerivedMeasurements(object):
+
+ @staticmethod
+ def process(measurements_csv):
+ raise NotImplementedError()
diff --git a/devlib/derived/derived_measurements.py b/devlib/derived/derived_measurements.py
new file mode 100644
index 0000000..770db88
--- /dev/null
+++ b/devlib/derived/derived_measurements.py
@@ -0,0 +1,97 @@
+# Copyright 2013-2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from __future__ import division
+from collections import defaultdict
+
+from devlib import DerivedMeasurements
+from devlib.instrument import Measurement, MEASUREMENT_TYPES, InstrumentChannel
+
+
+class DerivedEnergyMeasurements(DerivedMeasurements):
+
+ @staticmethod
+ def process(measurements_csv):
+
+ should_calculate_energy = []
+ use_timestamp = False
+
+ # Determine sites to calculate energy for
+ channel_map = defaultdict(list)
+ for channel in measurements_csv.channels:
+ channel_map[channel].append(channel.kind)
+ for channel, kinds in channel_map.iteritems():
+ if 'power' in kinds and not 'energy' in kinds:
+ should_calculate_energy.append(channel.site)
+ if channel.site == 'timestamp':
+ use_timestamp = True
+ time_measurment = channel.measurement_type
+
+ if measurements_csv.sample_rate_hz is None and not use_timestamp:
+ msg = 'Timestamp data is unavailable, please provide a sample rate'
+ raise ValueError(msg)
+
+ if use_timestamp:
+ # Find index of timestamp column
+ ts_index = [i for i, chan in enumerate(measurements_csv.channels)
+ if chan.site == 'timestamp']
+ if len(ts_index) > 1:
+ raise ValueError('Multiple timestamps detected')
+ ts_index = ts_index[0]
+
+ row_ts = 0
+ last_ts = 0
+ energy_results = defaultdict(dict)
+ power_results = defaultdict(float)
+
+ # Process data
+ for count, row in enumerate(measurements_csv.itermeasurements()):
+ if use_timestamp:
+ last_ts = row_ts
+ row_ts = time_measurment.convert(float(row[ts_index].value), 'time')
+ for entry in row:
+ channel = entry.channel
+ site = channel.site
+ if channel.kind == 'energy':
+ if count == 0:
+ energy_results[site]['start'] = entry.value
+ else:
+ energy_results[site]['end'] = entry.value
+
+ if channel.kind == 'power':
+ power_results[site] += entry.value
+
+ if site in should_calculate_energy:
+ if count == 0:
+ energy_results[site]['start'] = 0
+ energy_results[site]['end'] = 0
+ elif use_timestamp:
+ energy_results[site]['end'] += entry.value * (row_ts - last_ts)
+ else:
+ energy_results[site]['end'] += entry.value * (1 /
+ measurements_csv.sample_rate_hz)
+
+ # Calculate final measurements
+ derived_measurements = []
+ for site in energy_results:
+ total_energy = energy_results[site]['end'] - energy_results[site]['start']
+ instChannel = InstrumentChannel('cum_energy', site, MEASUREMENT_TYPES['energy'])
+ derived_measurements.append(Measurement(total_energy, instChannel))
+
+ for site in power_results:
+ power = power_results[site] / (count + 1) #pylint: disable=undefined-loop-variable
+ instChannel = InstrumentChannel('avg_power', site, MEASUREMENT_TYPES['power'])
+ derived_measurements.append(Measurement(power, instChannel))
+
+ return derived_measurements
diff --git a/devlib/instrument/__init__.py b/devlib/instrument/__init__.py
index 044c7d4..9f8ac00 100644
--- a/devlib/instrument/__init__.py
+++ b/devlib/instrument/__init__.py
@@ -48,6 +48,8 @@
if not isinstance(to, MeasurementType):
msg = 'Unexpected conversion target: "{}"'
raise ValueError(msg.format(to))
+ if to.name == self.name:
+ return value
if not to.name in self.conversions:
msg = 'No conversion from {} to {} available'
raise ValueError(msg.format(self.name, to.name))
@@ -75,12 +77,20 @@
MeasurementType('unknown', None),
MeasurementType('time', 'seconds',
conversions={
- 'time_us': lambda x: x * 1000,
+ 'time_us': lambda x: x * 1000000,
+ 'time_ms': lambda x: x * 1000,
}
),
MeasurementType('time_us', 'microseconds',
conversions={
+ 'time': lambda x: x / 1000000,
+ 'time_ms': lambda x: x / 1000,
+ }
+ ),
+ MeasurementType('time_ms', 'milliseconds',
+ conversions={
'time': lambda x: x / 1000,
+ 'time_us': lambda x: x * 1000,
}
),
MeasurementType('temperature', 'degrees'),
@@ -133,9 +143,10 @@
class MeasurementsCsv(object):
- def __init__(self, path, channels=None):
+ def __init__(self, path, channels=None, sample_rate_hz=None):
self.path = path
self.channels = channels
+ self.sample_rate_hz = sample_rate_hz
self._fh = open(path, 'rb')
if self.channels is None:
self._load_channels()
diff --git a/devlib/instrument/acmecape.py b/devlib/instrument/acmecape.py
new file mode 100644
index 0000000..e1bb6c1
--- /dev/null
+++ b/devlib/instrument/acmecape.py
@@ -0,0 +1,123 @@
+#pylint: disable=attribute-defined-outside-init
+from __future__ import division
+import csv
+import os
+import time
+import tempfile
+from fcntl import fcntl, F_GETFL, F_SETFL
+from string import Template
+from subprocess import Popen, PIPE, STDOUT
+
+from devlib import Instrument, CONTINUOUS, MeasurementsCsv
+from devlib.exception import HostError
+from devlib.utils.misc import which
+
+OUTPUT_CAPTURE_FILE = 'acme-cape.csv'
+IIOCAP_CMD_TEMPLATE = Template("""
+${iio_capture} -n ${host} -b ${buffer_size} -c -f ${outfile} ${iio_device}
+""")
+
+def _read_nonblock(pipe, size=1024):
+ fd = pipe.fileno()
+ flags = fcntl(fd, F_GETFL)
+ flags |= os.O_NONBLOCK
+ fcntl(fd, F_SETFL, flags)
+
+ output = ''
+ try:
+ while True:
+ output += pipe.read(size)
+ except IOError:
+ pass
+ return output
+
+
+class AcmeCapeInstrument(Instrument):
+
+ mode = CONTINUOUS
+
+ def __init__(self, target,
+ iio_capture=which('iio_capture'),
+ host='baylibre-acme.local',
+ iio_device='iio:device0',
+ buffer_size=256):
+ super(AcmeCapeInstrument, self).__init__(target)
+ self.iio_capture = iio_capture
+ self.host = host
+ self.iio_device = iio_device
+ self.buffer_size = buffer_size
+ self.sample_rate_hz = 100
+ if self.iio_capture is None:
+ raise HostError('Missing iio-capture binary')
+ self.command = None
+ self.process = None
+
+ self.add_channel('shunt', 'voltage')
+ self.add_channel('bus', 'voltage')
+ self.add_channel('device', 'power')
+ self.add_channel('device', 'current')
+ self.add_channel('timestamp', 'time_ms')
+
+ def reset(self, sites=None, kinds=None, channels=None):
+ super(AcmeCapeInstrument, self).reset(sites, kinds, channels)
+ self.raw_data_file = tempfile.mkstemp('.csv')[1]
+ params = dict(
+ iio_capture=self.iio_capture,
+ host=self.host,
+ buffer_size=self.buffer_size,
+ iio_device=self.iio_device,
+ outfile=self.raw_data_file
+ )
+ self.command = IIOCAP_CMD_TEMPLATE.substitute(**params)
+ self.logger.debug('ACME cape command: {}'.format(self.command))
+
+ def start(self):
+ self.process = Popen(self.command.split(), stdout=PIPE, stderr=STDOUT)
+
+ def stop(self):
+ self.process.terminate()
+ timeout_secs = 10
+ for _ in xrange(timeout_secs):
+ if self.process.poll() is not None:
+ break
+ time.sleep(1)
+ else:
+ output = _read_nonblock(self.process.stdout)
+ self.process.kill()
+ self.logger.error('iio-capture did not terminate gracefully')
+ if self.process.poll() is None:
+ msg = 'Could not terminate iio-capture:\n{}'
+ raise HostError(msg.format(output))
+ if not os.path.isfile(self.raw_data_file):
+ raise HostError('Output CSV not generated.')
+
+ def get_data(self, outfile):
+ if os.stat(self.raw_data_file).st_size == 0:
+ self.logger.warning('"{}" appears to be empty'.format(self.raw_data_file))
+ return
+
+ all_channels = [c.label for c in self.list_channels()]
+ active_channels = [c.label for c in self.active_channels]
+ active_indexes = [all_channels.index(ac) for ac in active_channels]
+
+ with open(self.raw_data_file, 'rb') as fh:
+ with open(outfile, 'wb') as wfh:
+ writer = csv.writer(wfh)
+ writer.writerow(active_channels)
+
+ reader = csv.reader(fh, skipinitialspace=True)
+ header = reader.next()
+ ts_index = header.index('timestamp ms')
+
+
+ for row in reader:
+ output_row = []
+ for i in active_indexes:
+ if i == ts_index:
+ # Leave time in ms
+ output_row.append(float(row[i]))
+ else:
+ # Convert rest into standard units.
+ output_row.append(float(row[i])/1000)
+ writer.writerow(output_row)
+ return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)
diff --git a/devlib/instrument/daq.py b/devlib/instrument/daq.py
index 58d2f3e..75e854d 100644
--- a/devlib/instrument/daq.py
+++ b/devlib/instrument/daq.py
@@ -126,7 +126,7 @@
writer.writerow(row)
raw_row = _read_next_rows()
- return MeasurementsCsv(outfile, self.active_channels)
+ return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)
finally:
for fh in file_handles:
fh.close()
diff --git a/devlib/instrument/energy_probe.py b/devlib/instrument/energy_probe.py
index ed502f5..5f47430 100644
--- a/devlib/instrument/energy_probe.py
+++ b/devlib/instrument/energy_probe.py
@@ -113,4 +113,4 @@
continue
else:
not_a_full_row_seen = True
- return MeasurementsCsv(outfile, self.active_channels)
+ return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)
diff --git a/devlib/instrument/frames.py b/devlib/instrument/frames.py
index d5a2147..d1899fb 100644
--- a/devlib/instrument/frames.py
+++ b/devlib/instrument/frames.py
@@ -16,6 +16,7 @@
self.collector_target = collector_target
self.period = period
self.keep_raw = keep_raw
+ self.sample_rate_hz = 1 / self.period
self.collector = None
self.header = None
self._need_reset = True
@@ -43,7 +44,7 @@
self.collector.process_frames(raw_outfile)
active_sites = [chan.label for chan in self.active_channels]
self.collector.write_frames(outfile, columns=active_sites)
- return MeasurementsCsv(outfile, self.active_channels)
+ return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)
def _init_channels(self):
raise NotImplementedError()
diff --git a/devlib/instrument/gem5power.py b/devlib/instrument/gem5power.py
index 7715ec1..d265440 100644
--- a/devlib/instrument/gem5power.py
+++ b/devlib/instrument/gem5power.py
@@ -28,10 +28,11 @@
mode = CONTINUOUS
roi_label = 'power_instrument'
-
+ site_mapping = {'timestamp': 'sim_seconds'}
+
def __init__(self, target, power_sites):
'''
- Parameter power_sites is a list of gem5 identifiers for power values.
+ Parameter power_sites is a list of gem5 identifiers for power values.
One example of such a field:
system.cluster0.cores0.power_model.static_power
'''
@@ -46,11 +47,14 @@
self.power_sites = power_sites
else:
self.power_sites = [power_sites]
- self.add_channel('sim_seconds', 'time')
+ self.add_channel('timestamp', 'time')
for field in self.power_sites:
self.add_channel(field, 'power')
self.target.gem5stats.book_roi(self.roi_label)
self.sample_period_ns = 10000000
+ # Sample rate must remain unset as gem5 does not provide samples
+ # at regular intervals therefore the reported timestamp should be used.
+ self.sample_rate_hz = None
self.target.gem5stats.start_periodic_dump(0, self.sample_period_ns)
self._base_stats_dump = 0
@@ -59,17 +63,18 @@
def stop(self):
self.target.gem5stats.roi_end(self.roi_label)
-
+
def get_data(self, outfile):
active_sites = [c.site for c in self.active_channels]
with open(outfile, 'wb') as wfh:
writer = csv.writer(wfh)
writer.writerow([c.label for c in self.active_channels]) # headers
- for rec, rois in self.target.gem5stats.match_iter(active_sites,
+ sites_to_match = [self.site_mapping.get(s, s) for s in active_sites]
+ for rec, rois in self.target.gem5stats.match_iter(sites_to_match,
[self.roi_label], self._base_stats_dump):
writer.writerow([float(rec[s]) for s in active_sites])
- return MeasurementsCsv(outfile, self.active_channels)
-
+ return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)
+
def reset(self, sites=None, kinds=None, channels=None):
super(Gem5PowerInstrument, self).reset(sites, kinds, channels)
self._base_stats_dump = self.target.gem5stats.next_dump_no()
diff --git a/devlib/instrument/monsoon.py b/devlib/instrument/monsoon.py
index e373d68..3103618 100644
--- a/devlib/instrument/monsoon.py
+++ b/devlib/instrument/monsoon.py
@@ -129,4 +129,4 @@
row.append(usb)
writer.writerow(row)
- return MeasurementsCsv(outfile, self.active_channels)
+ return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)
diff --git a/doc/derived_measurements.rst b/doc/derived_measurements.rst
new file mode 100644
index 0000000..fcd497c
--- /dev/null
+++ b/doc/derived_measurements.rst
@@ -0,0 +1,69 @@
+Derived Measurements
+=====================
+
+
+The ``DerivedMeasurements`` API provides a consistent way of performing post
+processing on a provided :class:`MeasurementCsv` file.
+
+Example
+-------
+
+The following example shows how to use an implementation of a
+:class:`DerivedMeasurement` to obtain a list of calculated ``Measurements``.
+
+.. code-block:: ipython
+
+ # Import the relevant derived measurement module
+ # in this example the derived energy module is used.
+ In [1]: from devlib import DerivedEnergyMeasurements
+
+ # Obtain a MeasurementCsv file from an instrument or create from
+ # existing .csv file. In this example an existing csv file is used which was
+ # created with a sampling rate of 100Hz
+ In [2]: from devlib import MeasurementsCsv
+ In [3]: measurement_csv = MeasurementsCsv('/example/measurements.csv', sample_rate_hz=100)
+
+ # Process the file and obtain a list of the derived measurements
+ In [4]: derived_measurements = DerivedEnergyMeasurements.process(measurement_csv)
+
+ In [5]: derived_measurements
+ Out[5]: [device_energy: 239.1854075 joules, device_power: 5.5494089227 watts]
+
+API
+---
+
+Derived Measurements
+~~~~~~~~~~~~~~~~~~~~
+
+.. class:: DerivedMeasurements()
+
+ The ``DerivedMeasurements`` class is an abstract base for implementing
+ additional classes to calculate various metrics.
+
+.. method:: DerivedMeasurements.process(measurement_csv)
+
+ Returns a list of :class:`Measurement` objects that have been calculated.
+
+
+
+Available Derived Measurements
+-------------------------------
+.. class:: DerivedEnergyMeasurements()
+
+ The ``DerivedEnergyMeasurements`` class is used to calculate average power and
+ cumulative energy for each site if the required data is present.
+
+ The calculation of cumulative energy can occur in 3 ways. If a
+ ``site`` contains ``energy`` results, the first and last measurements are extracted
+ and the delta calculated. If not, a ``timestamp`` channel will be used to calculate
+ the energy from the power channel, failing back to using the sample rate attribute
+ of the :class:`MeasurementCsv` file if timestamps are not available. If neither
+ timestamps or a sample rate are available then an error will be raised.
+
+
+.. method:: DerivedEnergyMeasurements.process(measurement_csv)
+
+ Returns a list of :class:`Measurement` objects that have been calculated for
+ the average power and cumulative energy for each site.
+
+
diff --git a/doc/index.rst b/doc/index.rst
index 2c6d72f..5f4dda5 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -19,6 +19,7 @@
target
modules
instrumentation
+ derived_measurements
platform
connection
diff --git a/doc/instrumentation.rst b/doc/instrumentation.rst
index 3c777ac..76adf39 100644
--- a/doc/instrumentation.rst
+++ b/doc/instrumentation.rst
@@ -139,6 +139,14 @@
``<site>_<kind>`` (see :class:`InstrumentChannel`). The order of the columns
will be the same as the order of channels in ``Instrument.active_channels``.
+ If reporting timestamps, one channel must have a ``site`` named ``"timestamp"``
+ and a ``kind`` of a :class:`MeasurmentType` of an appropriate time unit which will
+ be used, if appropriate, during any post processing.
+
+ .. note:: Currently supported time units are seconds, milliseconds and
+ microseconds, other units can also be used if an appropriate
+ conversion is provided.
+
This returns a :class:`MeasurementCsv` instance associated with the outfile
that can be used to stream :class:`Measurement`\ s lists (similar to what is
returned by ``take_measurement()``.
@@ -151,7 +159,7 @@
Sample rate of the instrument in Hz. Assumed to be the same for all channels.
.. note:: This attribute is only provided by :class:`Instrument`\ s that
- support ``CONTINUOUS`` measurment.
+ support ``CONTINUOUS`` measurement.
Instrument Channel
~~~~~~~~~~~~~~~~~~
@@ -211,27 +219,31 @@
defined measurement types are
-+-------------+---------+---------------+
-| name | units | category |
-+=============+=========+===============+
-| time | seconds | |
-+-------------+---------+---------------+
-| temperature | degrees | |
-+-------------+---------+---------------+
-| power | watts | power/energy |
-+-------------+---------+---------------+
-| voltage | volts | power/energy |
-+-------------+---------+---------------+
-| current | amps | power/energy |
-+-------------+---------+---------------+
-| energy | joules | power/energy |
-+-------------+---------+---------------+
-| tx | bytes | data transfer |
-+-------------+---------+---------------+
-| rx | bytes | data transfer |
-+-------------+---------+---------------+
-| tx/rx | bytes | data transfer |
-+-------------+---------+---------------+
++-------------+-------------+---------------+
+| name | units | category |
++=============+=============+===============+
+| time | seconds | |
++-------------+-------------+---------------+
+| time | microseconds| |
++-------------+-------------+---------------+
+| time | milliseconds| |
++-------------+-------------+---------------+
+| temperature | degrees | |
++-------------+-------------+---------------+
+| power | watts | power/energy |
++-------------+-------------+---------------+
+| voltage | volts | power/energy |
++-------------+-------------+---------------+
+| current | amps | power/energy |
++-------------+-------------+---------------+
+| energy | joules | power/energy |
++-------------+-------------+---------------+
+| tx | bytes | data transfer |
++-------------+-------------+---------------+
+| rx | bytes | data transfer |
++-------------+-------------+---------------+
+| tx/rx | bytes | data transfer |
++-------------+-------------+---------------+
.. instruments: