Merge pull request #146 from derkling/adb_addons
Add a couple of ADB related utility functions
diff --git a/devlib/__init__.py b/devlib/__init__.py
index 51a8e47..2f50632 100644
--- a/devlib/__init__.py
+++ b/devlib/__init__.py
@@ -13,9 +13,11 @@
from devlib.instrument import MEASUREMENT_TYPES, INSTANTANEOUS, CONTINUOUS
from devlib.instrument.daq import DaqInstrument
from devlib.instrument.energy_probe import EnergyProbeInstrument
+from devlib.instrument.frames import GfxInfoFramesInstrument
from devlib.instrument.hwmon import HwmonInstrument
from devlib.instrument.monsoon import MonsoonInstrument
from devlib.instrument.netstats import NetstatsInstrument
+from devlib.instrument.gem5power import Gem5PowerInstrument
from devlib.trace.ftrace import FtraceCollector
diff --git a/devlib/exception.py b/devlib/exception.py
index 16dd04f..11b19e0 100644
--- a/devlib/exception.py
+++ b/devlib/exception.py
@@ -13,7 +13,6 @@
# limitations under the License.
#
-
class DevlibError(Exception):
"""Base class for all Devlib exceptions."""
pass
@@ -49,3 +48,42 @@
def __str__(self):
return '\n'.join([self.message, 'OUTPUT:', self.output or ''])
+
+
+class WorkerThreadError(DevlibError):
+ """
+ This should get raised in the main thread if a non-WAError-derived
+ exception occurs on a worker/background thread. If a WAError-derived
+ exception is raised in the worker, then it that exception should be
+ re-raised on the main thread directly -- the main point of this is to
+ preserve the backtrace in the output, and backtrace doesn't get output for
+ WAErrors.
+
+ """
+
+ def __init__(self, thread, exc_info):
+ self.thread = thread
+ self.exc_info = exc_info
+ orig = self.exc_info[1]
+ orig_name = type(orig).__name__
+ message = 'Exception of type {} occured on thread {}:\n'.format(orig_name, thread)
+ message += '{}\n{}: {}'.format(get_traceback(self.exc_info), orig_name, orig)
+ super(WorkerThreadError, self).__init__(message)
+
+
+def get_traceback(exc=None):
+ """
+ Returns the string with the traceback for the specifiec exc
+ object, or for the current exception exc is not specified.
+
+ """
+ import StringIO, traceback, sys
+ if exc is None:
+ exc = sys.exc_info()
+ if not exc:
+ return None
+ tb = exc[2]
+ sio = StringIO.StringIO()
+ traceback.print_tb(tb, file=sio)
+ del tb # needs to be done explicitly see: http://docs.python.org/2/library/sys.html#sys.exc_info
+ return sio.getvalue()
diff --git a/devlib/instrument/__init__.py b/devlib/instrument/__init__.py
index 9d898c4..044c7d4 100644
--- a/devlib/instrument/__init__.py
+++ b/devlib/instrument/__init__.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+from __future__ import division
import csv
import logging
import collections
@@ -24,28 +25,33 @@
INSTANTANEOUS = 1
CONTINUOUS = 2
+MEASUREMENT_TYPES = {} # populated further down
-class MeasurementType(tuple):
- __slots__ = []
+class MeasurementType(object):
- def __new__(cls, name, units, category=None):
- return tuple.__new__(cls, (name, units, category))
+ def __init__(self, name, units, category=None, conversions=None):
+ self.name = name
+ self.units = units
+ self.category = category
+ self.conversions = {}
+ if conversions is not None:
+ for key, value in conversions.iteritems():
+ if not callable(value):
+ msg = 'Converter must be callable; got {} "{}"'
+ raise ValueError(msg.format(type(value), value))
+ self.conversions[key] = value
- @property
- def name(self):
- return tuple.__getitem__(self, 0)
-
- @property
- def units(self):
- return tuple.__getitem__(self, 1)
-
- @property
- def category(self):
- return tuple.__getitem__(self, 2)
-
- def __getitem__(self, item):
- raise TypeError()
+ def convert(self, value, to):
+ if isinstance(to, basestring) and to in MEASUREMENT_TYPES:
+ to = MEASUREMENT_TYPES[to]
+ if not isinstance(to, MeasurementType):
+ msg = 'Unexpected conversion target: "{}"'
+ raise ValueError(msg.format(to))
+ if not to.name in self.conversions:
+ msg = 'No conversion from {} to {} available'
+ raise ValueError(msg.format(self.name, to.name))
+ return self.conversions[to.name](value)
def __cmp__(self, other):
if isinstance(other, MeasurementType):
@@ -55,12 +61,28 @@
def __str__(self):
return self.name
- __repr__ = __str__
+ def __repr__(self):
+ if self.category:
+ text = 'MeasurementType({}, {}, {})'
+ return text.format(self.name, self.units, self.category)
+ else:
+ text = 'MeasurementType({}, {})'
+ return text.format(self.name, self.units)
# Standard measures
_measurement_types = [
- MeasurementType('time', 'seconds'),
+ MeasurementType('unknown', None),
+ MeasurementType('time', 'seconds',
+ conversions={
+ 'time_us': lambda x: x * 1000,
+ }
+ ),
+ MeasurementType('time_us', 'microseconds',
+ conversions={
+ 'time': lambda x: x / 1000,
+ }
+ ),
MeasurementType('temperature', 'degrees'),
MeasurementType('power', 'watts', 'power/energy'),
@@ -71,8 +93,11 @@
MeasurementType('tx', 'bytes', 'data transfer'),
MeasurementType('rx', 'bytes', 'data transfer'),
MeasurementType('tx/rx', 'bytes', 'data transfer'),
+
+ MeasurementType('frames', 'frames', 'ui render'),
]
-MEASUREMENT_TYPES = {m.name: m for m in _measurement_types}
+for m in _measurement_types:
+ MEASUREMENT_TYPES[m.name] = m
class Measurement(object):
@@ -108,10 +133,12 @@
class MeasurementsCsv(object):
- def __init__(self, path, channels):
+ def __init__(self, path, channels=None):
self.path = path
self.channels = channels
self._fh = open(path, 'rb')
+ if self.channels is None:
+ self._load_channels()
def measurements(self):
return list(self.itermeasurements())
@@ -124,6 +151,29 @@
values = map(numeric, row)
yield [Measurement(v, c) for (v, c) in zip(values, self.channels)]
+ def _load_channels(self):
+ self._fh.seek(0)
+ reader = csv.reader(self._fh)
+ header = reader.next()
+ self._fh.seek(0)
+
+ self.channels = []
+ for entry in header:
+ for mt in MEASUREMENT_TYPES:
+ suffix = '_{}'.format(mt)
+ if entry.endswith(suffix):
+ site = entry[:-len(suffix)]
+ measure = mt
+ name = '{}_{}'.format(site, measure)
+ break
+ else:
+ site = entry
+ measure = 'unknown'
+ name = entry
+
+ chan = InstrumentChannel(name, site, measure)
+ self.channels.append(chan)
+
class InstrumentChannel(object):
diff --git a/devlib/instrument/frames.py b/devlib/instrument/frames.py
new file mode 100644
index 0000000..d5a2147
--- /dev/null
+++ b/devlib/instrument/frames.py
@@ -0,0 +1,76 @@
+from devlib.instrument import (Instrument, CONTINUOUS,
+ MeasurementsCsv, MeasurementType)
+from devlib.utils.rendering import (GfxinfoFrameCollector,
+ SurfaceFlingerFrameCollector,
+ SurfaceFlingerFrame,
+ read_gfxinfo_columns)
+
+
+class FramesInstrument(Instrument):
+
+ mode = CONTINUOUS
+ collector_cls = None
+
+ def __init__(self, target, collector_target, period=2, keep_raw=True):
+ super(FramesInstrument, self).__init__(target)
+ self.collector_target = collector_target
+ self.period = period
+ self.keep_raw = keep_raw
+ self.collector = None
+ self.header = None
+ self._need_reset = True
+ self._init_channels()
+
+ def reset(self, sites=None, kinds=None, channels=None):
+ super(FramesInstrument, self).reset(sites, kinds, channels)
+ self.collector = self.collector_cls(self.target, self.period,
+ self.collector_target, self.header)
+ self._need_reset = False
+
+ def start(self):
+ if self._need_reset:
+ self.reset()
+ self.collector.start()
+
+ def stop(self):
+ self.collector.stop()
+ self._need_reset = True
+
+ def get_data(self, outfile):
+ raw_outfile = None
+ if self.keep_raw:
+ raw_outfile = outfile + '.raw'
+ self.collector.process_frames(raw_outfile)
+ active_sites = [chan.label for chan in self.active_channels]
+ self.collector.write_frames(outfile, columns=active_sites)
+ return MeasurementsCsv(outfile, self.active_channels)
+
+ def _init_channels(self):
+ raise NotImplementedError()
+
+
+class GfxInfoFramesInstrument(FramesInstrument):
+
+ mode = CONTINUOUS
+ collector_cls = GfxinfoFrameCollector
+
+ def _init_channels(self):
+ columns = read_gfxinfo_columns(self.target)
+ for entry in columns:
+ if entry == 'Flags':
+ self.add_channel('Flags', MeasurementType('flags', 'flags'))
+ else:
+ self.add_channel(entry, 'time_us')
+ self.header = [chan.label for chan in self.channels.values()]
+
+
+class SurfaceFlingerFramesInstrument(FramesInstrument):
+
+ mode = CONTINUOUS
+ collector_cls = SurfaceFlingerFrameCollector
+
+ def _init_channels(self):
+ for field in SurfaceFlingerFrame._fields:
+ # remove the "_time" from filed names to avoid duplication
+ self.add_channel(field[:-5], 'time_us')
+ self.header = [chan.label for chan in self.channels.values()]
diff --git a/devlib/instrument/gem5power.py b/devlib/instrument/gem5power.py
new file mode 100644
index 0000000..6411b3f
--- /dev/null
+++ b/devlib/instrument/gem5power.py
@@ -0,0 +1,74 @@
+# Copyright 2017 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import division
+import csv
+import re
+
+from devlib.platform.gem5 import Gem5SimulationPlatform
+from devlib.instrument import Instrument, CONTINUOUS, MeasurementsCsv
+from devlib.exception import TargetError, HostError
+
+
+class Gem5PowerInstrument(Instrument):
+ '''
+ Instrument enabling power monitoring in gem5
+ '''
+
+ mode = CONTINUOUS
+ roi_label = 'power_instrument'
+
+ def __init__(self, target, power_sites):
+ '''
+ Parameter power_sites is a list of gem5 identifiers for power values.
+ One example of such a field:
+ system.cluster0.cores0.power_model.static_power
+ '''
+ if not isinstance(target.platform, Gem5SimulationPlatform):
+ raise TargetError('Gem5PowerInstrument requires a gem5 platform')
+ if not target.has('gem5stats'):
+ raise TargetError('Gem5StatsModule is not loaded')
+ super(Gem5PowerInstrument, self).__init__(target)
+
+ # power_sites is assumed to be a list later
+ if isinstance(power_sites, list):
+ self.power_sites = power_sites
+ else:
+ self.power_sites = [power_sites]
+ self.add_channel('sim_seconds', 'time')
+ for field in self.power_sites:
+ self.add_channel(field, 'power')
+ self.target.gem5stats.book_roi(self.roi_label)
+ self.sample_period_ns = 10000000
+ self.target.gem5stats.start_periodic_dump(0, self.sample_period_ns)
+
+ def start(self):
+ self.target.gem5stats.roi_start(self.roi_label)
+
+ def stop(self):
+ self.target.gem5stats.roi_end(self.roi_label)
+
+ def get_data(self, outfile):
+ active_sites = [c.site for c in self.active_channels]
+ with open(outfile, 'wb') as wfh:
+ writer = csv.writer(wfh)
+ writer.writerow([c.label for c in self.active_channels]) # headers
+ for rec, rois in self.target.gem5stats.match_iter(active_sites, [self.roi_label]):
+ writer.writerow([float(rec[s]) for s in active_sites])
+ return MeasurementsCsv(outfile, self.active_channels)
+
+ def reset(self, sites=None, kinds=None, channels=None):
+ super(Gem5PowerInstrument, self).reset(sites, kinds, channels)
+ self.target.gem5stats.reset_origin()
+
diff --git a/devlib/module/biglittle.py b/devlib/module/biglittle.py
index eafcb0a..e353229 100644
--- a/devlib/module/biglittle.py
+++ b/devlib/module/biglittle.py
@@ -44,79 +44,151 @@
# cpufreq
def list_bigs_frequencies(self):
- return self.target.cpufreq.list_frequencies(self.bigs_online[0])
+ bigs_online = self.bigs_online
+ if len(bigs_online) > 0:
+ return self.target.cpufreq.list_frequencies(bigs_online[0])
def list_bigs_governors(self):
- return self.target.cpufreq.list_governors(self.bigs_online[0])
+ bigs_online = self.bigs_online
+ if len(bigs_online) > 0:
+ return self.target.cpufreq.list_governors(bigs_online[0])
def list_bigs_governor_tunables(self):
- return self.target.cpufreq.list_governor_tunables(self.bigs_online[0])
+ bigs_online = self.bigs_online
+ if len(bigs_online) > 0:
+ return self.target.cpufreq.list_governor_tunables(bigs_online[0])
def list_littles_frequencies(self):
- return self.target.cpufreq.list_frequencies(self.littles_online[0])
+ littles_online = self.littles_online
+ if len(littles_online) > 0:
+ return self.target.cpufreq.list_frequencies(littles_online[0])
def list_littles_governors(self):
- return self.target.cpufreq.list_governors(self.littles_online[0])
+ littles_online = self.littles_online
+ if len(littles_online) > 0:
+ return self.target.cpufreq.list_governors(littles_online[0])
def list_littles_governor_tunables(self):
- return self.target.cpufreq.list_governor_tunables(self.littles_online[0])
+ littles_online = self.littles_online
+ if len(littles_online) > 0:
+ return self.target.cpufreq.list_governor_tunables(littles_online[0])
def get_bigs_governor(self):
- return self.target.cpufreq.get_governor(self.bigs_online[0])
+ bigs_online = self.bigs_online
+ if len(bigs_online) > 0:
+ return self.target.cpufreq.get_governor(bigs_online[0])
def get_bigs_governor_tunables(self):
- return self.target.cpufreq.get_governor_tunables(self.bigs_online[0])
+ bigs_online = self.bigs_online
+ if len(bigs_online) > 0:
+ return self.target.cpufreq.get_governor_tunables(bigs_online[0])
def get_bigs_frequency(self):
- return self.target.cpufreq.get_frequency(self.bigs_online[0])
+ bigs_online = self.bigs_online
+ if len(bigs_online) > 0:
+ return self.target.cpufreq.get_frequency(bigs_online[0])
def get_bigs_min_frequency(self):
- return self.target.cpufreq.get_min_frequency(self.bigs_online[0])
+ bigs_online = self.bigs_online
+ if len(bigs_online) > 0:
+ return self.target.cpufreq.get_min_frequency(bigs_online[0])
def get_bigs_max_frequency(self):
- return self.target.cpufreq.get_max_frequency(self.bigs_online[0])
+ bigs_online = self.bigs_online
+ if len(bigs_online) > 0:
+ return self.target.cpufreq.get_max_frequency(bigs_online[0])
def get_littles_governor(self):
- return self.target.cpufreq.get_governor(self.littles_online[0])
+ littles_online = self.littles_online
+ if len(littles_online) > 0:
+ return self.target.cpufreq.get_governor(littles_online[0])
def get_littles_governor_tunables(self):
- return self.target.cpufreq.get_governor_tunables(self.littles_online[0])
+ littles_online = self.littles_online
+ if len(littles_online) > 0:
+ return self.target.cpufreq.get_governor_tunables(littles_online[0])
def get_littles_frequency(self):
- return self.target.cpufreq.get_frequency(self.littles_online[0])
+ littles_online = self.littles_online
+ if len(littles_online) > 0:
+ return self.target.cpufreq.get_frequency(littles_online[0])
def get_littles_min_frequency(self):
- return self.target.cpufreq.get_min_frequency(self.littles_online[0])
+ littles_online = self.littles_online
+ if len(littles_online) > 0:
+ return self.target.cpufreq.get_min_frequency(littles_online[0])
def get_littles_max_frequency(self):
- return self.target.cpufreq.get_max_frequency(self.littles_online[0])
+ littles_online = self.littles_online
+ if len(littles_online) > 0:
+ return self.target.cpufreq.get_max_frequency(littles_online[0])
def set_bigs_governor(self, governor, **kwargs):
- self.target.cpufreq.set_governor(self.bigs_online[0], governor, **kwargs)
+ bigs_online = self.bigs_online
+ if len(bigs_online) > 0:
+ self.target.cpufreq.set_governor(bigs_online[0], governor, **kwargs)
+ else:
+ raise ValueError("All bigs appear to be offline")
def set_bigs_governor_tunables(self, governor, **kwargs):
- self.target.cpufreq.set_governor_tunables(self.bigs_online[0], governor, **kwargs)
+ bigs_online = self.bigs_online
+ if len(bigs_online) > 0:
+ self.target.cpufreq.set_governor_tunables(bigs_online[0], governor, **kwargs)
+ else:
+ raise ValueError("All bigs appear to be offline")
def set_bigs_frequency(self, frequency, exact=True):
- self.target.cpufreq.set_frequency(self.bigs_online[0], frequency, exact)
+ bigs_online = self.bigs_online
+ if len(bigs_online) > 0:
+ self.target.cpufreq.set_frequency(bigs_online[0], frequency, exact)
+ else:
+ raise ValueError("All bigs appear to be offline")
def set_bigs_min_frequency(self, frequency, exact=True):
- self.target.cpufreq.set_min_frequency(self.bigs_online[0], frequency, exact)
+ bigs_online = self.bigs_online
+ if len(bigs_online) > 0:
+ self.target.cpufreq.set_min_frequency(bigs_online[0], frequency, exact)
+ else:
+ raise ValueError("All bigs appear to be offline")
def set_bigs_max_frequency(self, frequency, exact=True):
- self.target.cpufreq.set_max_frequency(self.bigs_online[0], frequency, exact)
+ bigs_online = self.bigs_online
+ if len(bigs_online) > 0:
+ self.target.cpufreq.set_max_frequency(bigs_online[0], frequency, exact)
+ else:
+ raise ValueError("All bigs appear to be offline")
def set_littles_governor(self, governor, **kwargs):
- self.target.cpufreq.set_governor(self.littles_online[0], governor, **kwargs)
+ littles_online = self.littles_online
+ if len(littles_online) > 0:
+ self.target.cpufreq.set_governor(littles_online[0], governor, **kwargs)
+ else:
+ raise ValueError("All littles appear to be offline")
def set_littles_governor_tunables(self, governor, **kwargs):
- self.target.cpufreq.set_governor_tunables(self.littles_online[0], governor, **kwargs)
+ littles_online = self.littles_online
+ if len(littles_online) > 0:
+ self.target.cpufreq.set_governor_tunables(littles_online[0], governor, **kwargs)
+ else:
+ raise ValueError("All littles appear to be offline")
def set_littles_frequency(self, frequency, exact=True):
- self.target.cpufreq.set_frequency(self.littles_online[0], frequency, exact)
+ littles_online = self.littles_online
+ if len(littles_online) > 0:
+ self.target.cpufreq.set_frequency(littles_online[0], frequency, exact)
+ else:
+ raise ValueError("All littles appear to be offline")
def set_littles_min_frequency(self, frequency, exact=True):
- self.target.cpufreq.set_min_frequency(self.littles_online[0], frequency, exact)
+ littles_online = self.littles_online
+ if len(littles_online) > 0:
+ self.target.cpufreq.set_min_frequency(littles_online[0], frequency, exact)
+ else:
+ raise ValueError("All littles appear to be offline")
def set_littles_max_frequency(self, frequency, exact=True):
- self.target.cpufreq.set_max_frequency(self.littles_online[0], frequency, exact)
+ littles_online = self.littles_online
+ if len(littles_online) > 0:
+ self.target.cpufreq.set_max_frequency(littles_online[0], frequency, exact)
+ else:
+ raise ValueError("All littles appear to be offline")
diff --git a/devlib/module/cpufreq.py b/devlib/module/cpufreq.py
index 01fb79b..e18d95b 100644
--- a/devlib/module/cpufreq.py
+++ b/devlib/module/cpufreq.py
@@ -382,7 +382,9 @@
'cpufreq_set_all_governors {}'.format(governor),
as_root=True)
except TargetError as e:
- if "echo: I/O error" in str(e):
+ if ("echo: I/O error" in str(e) or
+ "write error: Invalid argument" in str(e)):
+
cpus_unsupported = [c for c in self.target.list_online_cpus()
if governor not in self.list_governors(c)]
raise TargetError("Governor {} unsupported for CPUs {}".format(
diff --git a/devlib/module/gem5stats.py b/devlib/module/gem5stats.py
new file mode 100644
index 0000000..ba24a43
--- /dev/null
+++ b/devlib/module/gem5stats.py
@@ -0,0 +1,164 @@
+# Copyright 2017 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os.path
+from collections import defaultdict
+
+import devlib
+from devlib.module import Module
+from devlib.platform import Platform
+from devlib.platform.gem5 import Gem5SimulationPlatform
+from devlib.utils.gem5 import iter_statistics_dump, GEM5STATS_ROI_NUMBER, GEM5STATS_DUMP_TAIL
+
+
+class Gem5ROI:
+ def __init__(self, number, target):
+ self.target = target
+ self.number = number
+ self.running = False
+ self.field = 'ROI::{}'.format(number)
+
+ def start(self):
+ if self.running:
+ return False
+ self.target.execute('m5 roistart {}'.format(self.number))
+ self.running = True
+ return True
+
+ def stop(self):
+ if not self.running:
+ return False
+ self.target.execute('m5 roiend {}'.format(self.number))
+ self.running = False
+ return True
+
+class Gem5StatsModule(Module):
+ '''
+ Module controlling Region of Interest (ROIs) markers, satistics dump
+ frequency and parsing statistics log file when using gem5 platforms.
+
+ ROIs are identified by user-defined labels and need to be booked prior to
+ use. The translation of labels into gem5 ROI numbers will be performed
+ internally in order to avoid conflicts between multiple clients.
+ '''
+ name = 'gem5stats'
+
+ @staticmethod
+ def probe(target):
+ return isinstance(target.platform, Gem5SimulationPlatform)
+
+ def __init__(self, target):
+ super(Gem5StatsModule, self).__init__(target)
+ self._current_origin = 0
+ self._stats_file_path = os.path.join(target.platform.gem5_out_dir,
+ 'stats.txt')
+ self.rois = {}
+
+ def book_roi(self, label):
+ if label in self.rois:
+ raise KeyError('ROI label {} already used'.format(label))
+ if len(self.rois) >= GEM5STATS_ROI_NUMBER:
+ raise RuntimeError('Too many ROIs reserved')
+ all_rois = set(xrange(GEM5STATS_ROI_NUMBER))
+ used_rois = set([roi.number for roi in self.rois.values()])
+ avail_rois = all_rois - used_rois
+ self.rois[label] = Gem5ROI(list(avail_rois)[0], self.target)
+
+ def free_roi(self, label):
+ if label not in self.rois:
+ raise KeyError('ROI label {} not reserved yet'.format(label))
+ self.rois[label].stop()
+ del self.rois[label]
+
+ def roi_start(self, label):
+ if label not in self.rois:
+ raise KeyError('Incorrect ROI label: {}'.format(label))
+ if not self.rois[label].start():
+ raise TargetError('ROI {} was already running'.format(label))
+
+ def roi_end(self, label):
+ if label not in self.rois:
+ raise KeyError('Incorrect ROI label: {}'.format(label))
+ if not self.rois[label].stop():
+ raise TargetError('ROI {} was not running'.format(label))
+
+ def start_periodic_dump(self, delay_ns=0, period_ns=10000000):
+ # Default period is 10ms because it's roughly what's needed to have
+ # accurate power estimations
+ if delay_ns < 0 or period_ns < 0:
+ msg = 'Delay ({}) and period ({}) for periodic dumps must be positive'
+ raise ValueError(msg.format(delay_ns, period_ns))
+ self.target.execute('m5 dumpresetstats {} {}'.format(delay_ns, period_ns))
+
+ def match(self, keys, rois_labels):
+ '''
+ Tries to match the list of keys passed as parameter over the statistics
+ dumps covered by selected ROIs since origin. Returns a dict indexed by
+ key parameters containing a dict indexed by ROI labels containing an
+ in-order list of records for the key under consideration during the
+ active intervals of the ROI.
+
+ Keys must match fields in gem5's statistics log file. Key example:
+ system.cluster0.cores0.power_model.static_power
+ '''
+ records = defaultdict(lambda : defaultdict(list))
+ for record, active_rois in self.match_iter(keys, rois_labels):
+ for key in record:
+ for roi_label in active_rois:
+ records[key][roi_label].append(record[key])
+ return records
+
+ def match_iter(self, keys, rois_labels):
+ '''
+ Yields for each dump since origin a pair containing:
+ 1. a dict storing the values corresponding to each of the specified keys
+ 2. the list of currently active ROIs among those passed as parameters.
+
+ Keys must match fields in gem5's statistics log file. Key example:
+ system.cluster0.cores0.power_model.static_power
+ '''
+ for label in rois_labels:
+ if label not in self.rois:
+ raise KeyError('Impossible to match ROI label {}'.format(label))
+ if self.rois[label].running:
+ self.logger.warning('Trying to match records in statistics file'
+ ' while ROI {} is running'.format(label))
+
+ def roi_active(roi_label, dump):
+ roi = self.rois[roi_label]
+ return (roi.field in dump) and (int(dump[roi.field]) == 1)
+
+ with open(self._stats_file_path, 'r') as stats_file:
+ stats_file.seek(self._current_origin)
+ for dump in iter_statistics_dump(stats_file):
+ active_rois = [l for l in rois_labels if roi_active(l, dump)]
+ if active_rois:
+ record = {k: dump[k] for k in keys}
+ yield (record, active_rois)
+
+
+ def reset_origin(self):
+ '''
+ Place origin right after the last full dump in the file
+ '''
+ last_dump_tail = self._current_origin
+ # Dump & reset stats to start from a fresh state
+ self.target.execute('m5 dumpresetstats')
+ with open(self._stats_file_path, 'r') as stats_file:
+ for line in stats_file:
+ if GEM5STATS_DUMP_TAIL in line:
+ last_dump_tail = stats_file.tell()
+ self._current_origin = last_dump_tail
+
diff --git a/devlib/target.py b/devlib/target.py
index e9b24df..a920655 100644
--- a/devlib/target.py
+++ b/devlib/target.py
@@ -351,6 +351,38 @@
command = 'cd {} && {}'.format(in_directory, command)
return self.execute(command, as_root=as_root, timeout=timeout)
+ def background_invoke(self, binary, args=None, in_directory=None,
+ on_cpus=None, as_root=False):
+ """
+ Executes the specified binary as a background task under the
+ specified conditions.
+
+ :binary: binary to execute. Must be present and executable on the device.
+ :args: arguments to be passed to the binary. The can be either a list or
+ a string.
+ :in_directory: execute the binary in the specified directory. This must
+ be an absolute path.
+ :on_cpus: taskset the binary to these CPUs. This may be a single ``int`` (in which
+ case, it will be interpreted as the mask), a list of ``ints``, in which
+ case this will be interpreted as the list of cpus, or string, which
+ will be interpreted as a comma-separated list of cpu ranges, e.g.
+ ``"0,4-7"``.
+ :as_root: Specify whether the command should be run as root
+
+ :returns: the subprocess instance handling that command
+ """
+ command = binary
+ if args:
+ if isiterable(args):
+ args = ' '.join(args)
+ command = '{} {}'.format(command, args)
+ if on_cpus:
+ on_cpus = bitmask(on_cpus)
+ command = '{} taskset 0x{:x} {}'.format(self.busybox, on_cpus, command)
+ if in_directory:
+ command = 'cd {} && {}'.format(in_directory, command)
+ return self.background(command, as_root=as_root)
+
def kick_off(self, command, as_root=False):
raise NotImplementedError()
@@ -1040,10 +1072,24 @@
return line.split('=', 1)[1]
return None
- def install_apk(self, filepath, timeout=None): # pylint: disable=W0221
+ def get_sdk_version(self):
+ try:
+ return int(self.getprop('ro.build.version.sdk'))
+ except (ValueError, TypeError):
+ return None
+
+ def install_apk(self, filepath, timeout=None, replace=False, allow_downgrade=False): # pylint: disable=W0221
ext = os.path.splitext(filepath)[1].lower()
if ext == '.apk':
- return adb_command(self.adb_name, "install '{}'".format(filepath), timeout=timeout)
+ flags = []
+ if replace:
+ flags.append('-r') # Replace existing APK
+ if allow_downgrade:
+ flags.append('-d') # Install the APK even if a newer version is already installed
+ if self.get_sdk_version() >= 23:
+ flags.append('-g') # Grant all runtime permissions
+ self.logger.debug("Replace APK = {}, ADB flags = '{}'".format(replace, ' '.join(flags)))
+ return adb_command(self.adb_name, "install {} '{}'".format(' '.join(flags), filepath), timeout=timeout)
else:
raise TargetError('Can\'t install {}: unsupported format.'.format(filepath))
diff --git a/devlib/utils/android.py b/devlib/utils/android.py
index bd49ea4..f683190 100644
--- a/devlib/utils/android.py
+++ b/devlib/utils/android.py
@@ -34,7 +34,7 @@
logger = logging.getLogger('android')
MAX_ATTEMPTS = 5
-AM_START_ERROR = re.compile(r"Error: Activity class {[\w|.|/]*} does not exist")
+AM_START_ERROR = re.compile(r"Error: Activity.*")
# See:
# http://developer.android.com/guide/topics/manifest/uses-sdk-element.html#ApiLevels
@@ -366,19 +366,19 @@
if check_exit_code:
exit_code = exit_code.strip()
+ re_search = AM_START_ERROR.findall('{}\n{}'.format(output, error))
if exit_code.isdigit():
if int(exit_code):
message = ('Got exit code {}\nfrom target command: {}\n'
'STDOUT: {}\nSTDERR: {}')
raise TargetError(message.format(exit_code, command, output, error))
- elif AM_START_ERROR.findall(output):
- message = 'Could not start activity; got the following:'
- message += '\n{}'.format(AM_START_ERROR.findall(output)[0])
- raise TargetError(message)
- else: # not all digits
- if AM_START_ERROR.findall(output):
+ elif re_search:
message = 'Could not start activity; got the following:\n{}'
- raise TargetError(message.format(AM_START_ERROR.findall(output)[0]))
+ raise TargetError(message.format(re_search[0]))
+ else: # not all digits
+ if re_search:
+ message = 'Could not start activity; got the following:\n{}'
+ raise TargetError(message.format(re_search[0]))
else:
message = 'adb has returned early; did not get an exit code. '\
'Was kill-server invoked?'
diff --git a/devlib/utils/gem5.py b/devlib/utils/gem5.py
new file mode 100644
index 0000000..c609d70
--- /dev/null
+++ b/devlib/utils/gem5.py
@@ -0,0 +1,43 @@
+# Copyright 2017 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+
+GEM5STATS_FIELD_REGEX = re.compile("^(?P<key>[^- ]\S*) +(?P<value>[^#]+).+$")
+GEM5STATS_DUMP_HEAD = '---------- Begin Simulation Statistics ----------'
+GEM5STATS_DUMP_TAIL = '---------- End Simulation Statistics ----------'
+GEM5STATS_ROI_NUMBER = 8
+
+
+def iter_statistics_dump(stats_file):
+ '''
+ Yields statistics dumps as dicts. The parameter is assumed to be a stream
+ reading from the statistics log file.
+ '''
+ cur_dump = {}
+ while True:
+ line = stats_file.readline()
+ if not line:
+ break
+ if GEM5STATS_DUMP_TAIL in line:
+ yield cur_dump
+ cur_dump = {}
+ else:
+ res = GEM5STATS_FIELD_REGEX.match(line)
+ if res:
+ k = res.group("key")
+ v = res.group("value").split()
+ cur_dump[k] = v[0] if len(v)==1 else set(v)
+
diff --git a/devlib/utils/misc.py b/devlib/utils/misc.py
index b8626aa..d6a5093 100644
--- a/devlib/utils/misc.py
+++ b/devlib/utils/misc.py
@@ -79,9 +79,19 @@
0xd08: {None: 'A72'},
0xd09: {None: 'A73'},
},
+ 0x42: { # Broadcom
+ 0x516: {None: 'Vulcan'},
+ },
+ 0x43: { # Cavium
+ 0x0a1: {None: 'Thunderx'},
+ 0x0a2: {None: 'Thunderx81xx'},
+ },
0x4e: { # Nvidia
0x0: {None: 'Denver'},
},
+ 0x50: { # AppliedMicro
+ 0x0: {None: 'xgene'},
+ },
0x51: { # Qualcomm
0x02d: {None: 'Scorpion'},
0x04d: {None: 'MSM8960'},
@@ -91,6 +101,7 @@
},
0x205: {0x1: 'KryoSilver'},
0x211: {0x1: 'KryoGold'},
+ 0x800: {None: 'Falkor'},
},
0x56: { # Marvell
0x131: {
diff --git a/devlib/utils/rendering.py b/devlib/utils/rendering.py
new file mode 100644
index 0000000..3b7b6c4
--- /dev/null
+++ b/devlib/utils/rendering.py
@@ -0,0 +1,205 @@
+import csv
+import logging
+import os
+import re
+import shutil
+import sys
+import tempfile
+import threading
+import time
+from collections import namedtuple, OrderedDict
+from distutils.version import LooseVersion
+
+from devlib.exception import WorkerThreadError, TargetNotRespondingError, TimeoutError
+
+
+logger = logging.getLogger('rendering')
+
+SurfaceFlingerFrame = namedtuple('SurfaceFlingerFrame',
+ 'desired_present_time actual_present_time frame_ready_time')
+
+
+class FrameCollector(threading.Thread):
+
+ def __init__(self, target, period):
+ super(FrameCollector, self).__init__()
+ self.target = target
+ self.period = period
+ self.stop_signal = threading.Event()
+ self.frames = []
+
+ self.temp_file = None
+ self.refresh_period = None
+ self.drop_threshold = None
+ self.unresponsive_count = 0
+ self.last_ready_time = None
+ self.exc = None
+ self.header = None
+
+ def run(self):
+ logger.debug('Surface flinger frame data collection started.')
+ try:
+ self.stop_signal.clear()
+ fd, self.temp_file = tempfile.mkstemp()
+ logger.debug('temp file: {}'.format(self.temp_file))
+ wfh = os.fdopen(fd, 'wb')
+ try:
+ while not self.stop_signal.is_set():
+ self.collect_frames(wfh)
+ time.sleep(self.period)
+ finally:
+ wfh.close()
+ except (TargetNotRespondingError, TimeoutError): # pylint: disable=W0703
+ raise
+ except Exception, e: # pylint: disable=W0703
+ logger.warning('Exception on collector thread: {}({})'.format(e.__class__.__name__, e))
+ self.exc = WorkerThreadError(self.name, sys.exc_info())
+ logger.debug('Surface flinger frame data collection stopped.')
+
+ def stop(self):
+ self.stop_signal.set()
+ self.join()
+ if self.unresponsive_count:
+ message = 'FrameCollector was unrepsonsive {} times.'.format(self.unresponsive_count)
+ if self.unresponsive_count > 10:
+ logger.warning(message)
+ else:
+ logger.debug(message)
+ if self.exc:
+ raise self.exc # pylint: disable=E0702
+
+ def process_frames(self, outfile=None):
+ if not self.temp_file:
+ raise RuntimeError('Attempting to process frames before running the collector')
+ with open(self.temp_file) as fh:
+ self._process_raw_file(fh)
+ if outfile:
+ shutil.copy(self.temp_file, outfile)
+ os.unlink(self.temp_file)
+ self.temp_file = None
+
+ def write_frames(self, outfile, columns=None):
+ if columns is None:
+ header = self.header
+ frames = self.frames
+ else:
+ header = [c for c in self.header if c in columns]
+ indexes = [self.header.index(c) for c in header]
+ frames = [[f[i] for i in indexes] for f in self.frames]
+ with open(outfile, 'w') as wfh:
+ writer = csv.writer(wfh)
+ if header:
+ writer.writerow(header)
+ writer.writerows(frames)
+
+ def collect_frames(self, wfh):
+ raise NotImplementedError()
+
+ def clear(self):
+ raise NotImplementedError()
+
+ def _process_raw_file(self, fh):
+ raise NotImplementedError()
+
+
+class SurfaceFlingerFrameCollector(FrameCollector):
+
+ def __init__(self, target, period, view, header=None):
+ super(SurfaceFlingerFrameCollector, self).__init__(target, period)
+ self.view = view
+ self.header = header or SurfaceFlingerFrame._fields
+
+ def collect_frames(self, wfh):
+ for activity in self.list():
+ if activity == self.view:
+ wfh.write(self.get_latencies(activity))
+
+ def clear(self):
+ self.target.execute('dumpsys SurfaceFlinger --latency-clear ')
+
+ def get_latencies(self, activity):
+ cmd = 'dumpsys SurfaceFlinger --latency "{}"'
+ return self.target.execute(cmd.format(activity))
+
+ def list(self):
+ return self.target.execute('dumpsys SurfaceFlinger --list').split('\r\n')
+
+ def _process_raw_file(self, fh):
+ text = fh.read().replace('\r\n', '\n').replace('\r', '\n')
+ for line in text.split('\n'):
+ line = line.strip()
+ if line:
+ self._process_trace_line(line)
+
+ def _process_trace_line(self, line):
+ parts = line.split()
+ if len(parts) == 3:
+ frame = SurfaceFlingerFrame(*map(int, parts))
+ if not frame.frame_ready_time:
+ return # "null" frame
+ if frame.frame_ready_time <= self.last_ready_time:
+ return # duplicate frame
+ if (frame.frame_ready_time - frame.desired_present_time) > self.drop_threshold:
+ logger.debug('Dropping bogus frame {}.'.format(line))
+ return # bogus data
+ self.last_ready_time = frame.frame_ready_time
+ self.frames.append(frame)
+ elif len(parts) == 1:
+ self.refresh_period = int(parts[0])
+ self.drop_threshold = self.refresh_period * 1000
+ elif 'SurfaceFlinger appears to be unresponsive, dumping anyways' in line:
+ self.unresponsive_count += 1
+ else:
+ logger.warning('Unexpected SurfaceFlinger dump output: {}'.format(line))
+
+
+def read_gfxinfo_columns(target):
+ output = target.execute('dumpsys gfxinfo --list framestats')
+ lines = iter(output.split('\n'))
+ for line in lines:
+ if line.startswith('---PROFILEDATA---'):
+ break
+ columns_line = lines.next()
+ return columns_line.split(',')[:-1] # has a trailing ','
+
+
+class GfxinfoFrameCollector(FrameCollector):
+
+ def __init__(self, target, period, package, header=None):
+ super(GfxinfoFrameCollector, self).__init__(target, period)
+ self.package = package
+ self.header = None
+ self._init_header(header)
+
+ def collect_frames(self, wfh):
+ cmd = 'dumpsys gfxinfo {} framestats'
+ wfh.write(self.target.execute(cmd.format(self.package)))
+
+ def clear(self):
+ pass
+
+ def _init_header(self, header):
+ if header is not None:
+ self.header = header
+ else:
+ self.header = read_gfxinfo_columns(self.target)
+
+ def _process_raw_file(self, fh):
+ found = False
+ try:
+ while True:
+ for line in fh:
+ if line.startswith('---PROFILEDATA---'):
+ found = True
+ break
+
+ fh.next() # headers
+ for line in fh:
+ if line.startswith('---PROFILEDATA---'):
+ break
+ self.frames.append(map(int, line.strip().split(',')[:-1])) # has a trailing ','
+ except StopIteration:
+ pass
+ if not found:
+ logger.warning('Could not find frames data in gfxinfo output')
+ return
diff --git a/devlib/utils/ssh.py b/devlib/utils/ssh.py
index 8704008..89613bd 100644
--- a/devlib/utils/ssh.py
+++ b/devlib/utils/ssh.py
@@ -160,7 +160,8 @@
telnet=False,
password_prompt=None,
original_prompt=None,
- platform=None
+ platform=None,
+ sudo_cmd="sudo -- sh -c '{}'"
):
self.host = host
self.username = username
@@ -169,6 +170,7 @@
self.port = port
self.lock = threading.Lock()
self.password_prompt = password_prompt if password_prompt is not None else self.default_password_prompt
+ self.sudo_cmd = sudo_cmd
logger.debug('Logging in {}@{}'.format(username, host))
timeout = timeout if timeout is not None else self.default_timeout
self.conn = ssh_get_shell(host, username, password, self.keyfile, port, timeout, False, None)
@@ -212,7 +214,7 @@
port_string = '-p {}'.format(self.port) if self.port else ''
keyfile_string = '-i {}'.format(self.keyfile) if self.keyfile else ''
if as_root:
- command = "sudo -- sh -c '{}'".format(command)
+ command = self.sudo_cmd.format(command)
command = '{} {} {} {}@{} {}'.format(ssh, keyfile_string, port_string, self.username, self.host, command)
logger.debug(command)
if self.password:
@@ -240,7 +242,7 @@
# As we're already root, there is no need to use sudo.
as_root = False
if as_root:
- command = "sudo -- sh -c '{}'".format(escape_single_quotes(command))
+ command = self.sudo_cmd.format(escape_single_quotes(command))
if log:
logger.debug(command)
self.conn.sendline(command)
diff --git a/doc/target.rst b/doc/target.rst
index d14942e..5339a72 100644
--- a/doc/target.rst
+++ b/doc/target.rst
@@ -265,6 +265,24 @@
:param timeout: If this is specified and invocation does not terminate within this number
of seconds, an exception will be raised.
+.. method:: Target.background_invoke(binary [, args [, in_directory [, on_cpus [, as_root ]]]])
+
+ Execute the specified binary on target (must already be installed) as a background
+ task, under the specified conditions and return the :class:`subprocess.Popen`
+ instance for the command.
+
+ :param binary: binary to execute. Must be present and executable on the device.
+ :param args: arguments to be passed to the binary. The can be either a list or
+ a string.
+ :param in_directory: execute the binary in the specified directory. This must
+ be an absolute path.
+ :param on_cpus: taskset the binary to these CPUs. This may be a single ``int`` (in which
+ case, it will be interpreted as the mask), a list of ``ints``, in which
+ case this will be interpreted as the list of cpus, or string, which
+ will be interpreted as a comma-separated list of cpu ranges, e.g.
+ ``"0,4-7"``.
+ :param as_root: Specify whether the command should be run as root
+
.. method:: Target.kick_off(command [, as_root])
Kick off the specified command on the target and return immediately. Unlike