Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 1 | # Copyright 2013-2015 ARM Limited |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | # |
| 15 | |
| 16 | |
| 17 | """ |
| 18 | Miscellaneous functions that don't fit anywhere else. |
| 19 | |
| 20 | """ |
| 21 | from __future__ import division |
| 22 | import os |
| 23 | import sys |
| 24 | import re |
| 25 | import string |
| 26 | import threading |
| 27 | import signal |
| 28 | import subprocess |
| 29 | import pkgutil |
| 30 | import logging |
| 31 | import random |
Sergei Trofimov | 6d854fd | 2016-09-06 09:57:58 +0100 | [diff] [blame] | 32 | import ctypes |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 33 | from operator import itemgetter |
| 34 | from itertools import groupby |
| 35 | from functools import partial |
| 36 | |
Michele Di Giorgio | 539e9b3 | 2016-06-22 17:54:59 +0100 | [diff] [blame] | 37 | import wrapt |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 38 | |
Sergei Trofimov | 28891a8 | 2017-02-08 11:21:06 +0000 | [diff] [blame] | 39 | from devlib.exception import HostError, TimeoutError |
Sergei Trofimov | a926503 | 2017-02-08 11:14:40 +0000 | [diff] [blame] | 40 | |
| 41 | |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 42 | # ABI --> architectures list |
| 43 | ABI_MAP = { |
Marc Bonnici | 02c93b4 | 2017-07-17 15:18:39 +0100 | [diff] [blame] | 44 | 'armeabi': ['armeabi', 'armv7', 'armv7l', 'armv7el', 'armv7lh', 'armeabi-v7a'], |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 45 | 'arm64': ['arm64', 'armv8', 'arm64-v8a', 'aarch64'], |
| 46 | } |
| 47 | |
| 48 | # Vendor ID --> CPU part ID --> CPU variant ID --> Core Name |
| 49 | # None means variant is not used. |
| 50 | CPU_PART_MAP = { |
| 51 | 0x41: { # ARM |
| 52 | 0x926: {None: 'ARM926'}, |
| 53 | 0x946: {None: 'ARM946'}, |
| 54 | 0x966: {None: 'ARM966'}, |
| 55 | 0xb02: {None: 'ARM11MPCore'}, |
| 56 | 0xb36: {None: 'ARM1136'}, |
| 57 | 0xb56: {None: 'ARM1156'}, |
| 58 | 0xb76: {None: 'ARM1176'}, |
| 59 | 0xc05: {None: 'A5'}, |
| 60 | 0xc07: {None: 'A7'}, |
| 61 | 0xc08: {None: 'A8'}, |
| 62 | 0xc09: {None: 'A9'}, |
Sergei Trofimov | 08b36e7 | 2016-09-06 16:01:09 +0100 | [diff] [blame] | 63 | 0xc0e: {None: 'A17'}, |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 64 | 0xc0f: {None: 'A15'}, |
| 65 | 0xc14: {None: 'R4'}, |
| 66 | 0xc15: {None: 'R5'}, |
Sergei Trofimov | 08b36e7 | 2016-09-06 16:01:09 +0100 | [diff] [blame] | 67 | 0xc17: {None: 'R7'}, |
| 68 | 0xc18: {None: 'R8'}, |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 69 | 0xc20: {None: 'M0'}, |
Sergei Trofimov | 08b36e7 | 2016-09-06 16:01:09 +0100 | [diff] [blame] | 70 | 0xc60: {None: 'M0+'}, |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 71 | 0xc21: {None: 'M1'}, |
| 72 | 0xc23: {None: 'M3'}, |
| 73 | 0xc24: {None: 'M4'}, |
| 74 | 0xc27: {None: 'M7'}, |
Sergei Trofimov | 08b36e7 | 2016-09-06 16:01:09 +0100 | [diff] [blame] | 75 | 0xd01: {None: 'A32'}, |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 76 | 0xd03: {None: 'A53'}, |
Sergei Trofimov | 08b36e7 | 2016-09-06 16:01:09 +0100 | [diff] [blame] | 77 | 0xd04: {None: 'A35'}, |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 78 | 0xd07: {None: 'A57'}, |
| 79 | 0xd08: {None: 'A72'}, |
Sergei Trofimov | 08b36e7 | 2016-09-06 16:01:09 +0100 | [diff] [blame] | 80 | 0xd09: {None: 'A73'}, |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 81 | }, |
Sergei Trofimov | 86c9b6a | 2017-07-10 14:54:14 +0100 | [diff] [blame] | 82 | 0x42: { # Broadcom |
| 83 | 0x516: {None: 'Vulcan'}, |
Ionela Voinescu | 0a95bbe | 2017-07-10 16:06:24 +0100 | [diff] [blame] | 84 | }, |
Sergei Trofimov | 86c9b6a | 2017-07-10 14:54:14 +0100 | [diff] [blame] | 85 | 0x43: { # Cavium |
| 86 | 0x0a1: {None: 'Thunderx'}, |
| 87 | 0x0a2: {None: 'Thunderx81xx'}, |
| 88 | }, |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 89 | 0x4e: { # Nvidia |
| 90 | 0x0: {None: 'Denver'}, |
| 91 | }, |
Sergei Trofimov | 86c9b6a | 2017-07-10 14:54:14 +0100 | [diff] [blame] | 92 | 0x50: { # AppliedMicro |
| 93 | 0x0: {None: 'xgene'}, |
| 94 | }, |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 95 | 0x51: { # Qualcomm |
| 96 | 0x02d: {None: 'Scorpion'}, |
| 97 | 0x04d: {None: 'MSM8960'}, |
| 98 | 0x06f: { # Krait |
| 99 | 0x2: 'Krait400', |
| 100 | 0x3: 'Krait450', |
| 101 | }, |
Sergei Trofimov | 5d492ca | 2016-11-29 13:09:46 +0000 | [diff] [blame] | 102 | 0x205: {0x1: 'KryoSilver'}, |
| 103 | 0x211: {0x1: 'KryoGold'}, |
Sergei Trofimov | 86c9b6a | 2017-07-10 14:54:14 +0100 | [diff] [blame] | 104 | 0x800: {None: 'Falkor'}, |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 105 | }, |
Sergei Trofimov | 15333eb | 2017-08-22 09:27:24 +0100 | [diff] [blame] | 106 | 0x53: { # Samsung LSI |
| 107 | 0x001: {0x1: 'MongooseM1'}, |
| 108 | }, |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 109 | 0x56: { # Marvell |
| 110 | 0x131: { |
| 111 | 0x2: 'Feroceon 88F6281', |
| 112 | } |
| 113 | }, |
| 114 | } |
| 115 | |
| 116 | |
| 117 | def get_cpu_name(implementer, part, variant): |
| 118 | part_data = CPU_PART_MAP.get(implementer, {}).get(part, {}) |
| 119 | if None in part_data: # variant does not determine core Name for this vendor |
| 120 | name = part_data[None] |
| 121 | else: |
| 122 | name = part_data.get(variant) |
| 123 | return name |
| 124 | |
| 125 | |
| 126 | def preexec_function(): |
| 127 | # Ignore the SIGINT signal by setting the handler to the standard |
| 128 | # signal handler SIG_IGN. |
| 129 | signal.signal(signal.SIGINT, signal.SIG_IGN) |
| 130 | # Change process group in case we have to kill the subprocess and all of |
| 131 | # its children later. |
| 132 | # TODO: this is Unix-specific; would be good to find an OS-agnostic way |
| 133 | # to do this in case we wanna port WA to Windows. |
| 134 | os.setpgrp() |
| 135 | |
| 136 | |
| 137 | check_output_logger = logging.getLogger('check_output') |
| 138 | |
| 139 | |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 140 | def check_output(command, timeout=None, ignore=None, inputtext=None, **kwargs): |
| 141 | """This is a version of subprocess.check_output that adds a timeout parameter to kill |
| 142 | the subprocess if it does not return within the specified time.""" |
| 143 | # pylint: disable=too-many-branches |
| 144 | if ignore is None: |
| 145 | ignore = [] |
| 146 | elif isinstance(ignore, int): |
| 147 | ignore = [ignore] |
| 148 | elif not isinstance(ignore, list) and ignore != 'all': |
| 149 | message = 'Invalid value for ignore parameter: "{}"; must be an int or a list' |
| 150 | raise ValueError(message.format(ignore)) |
| 151 | if 'stdout' in kwargs: |
| 152 | raise ValueError('stdout argument not allowed, it will be overridden.') |
| 153 | |
| 154 | def callback(pid): |
| 155 | try: |
| 156 | check_output_logger.debug('{} timed out; sending SIGKILL'.format(pid)) |
| 157 | os.killpg(pid, signal.SIGKILL) |
| 158 | except OSError: |
| 159 | pass # process may have already terminated. |
| 160 | |
| 161 | process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, |
| 162 | stdin=subprocess.PIPE, |
| 163 | preexec_fn=preexec_function, **kwargs) |
| 164 | |
| 165 | if timeout: |
| 166 | timer = threading.Timer(timeout, callback, [process.pid, ]) |
| 167 | timer.start() |
| 168 | |
| 169 | try: |
| 170 | output, error = process.communicate(inputtext) |
| 171 | finally: |
| 172 | if timeout: |
| 173 | timer.cancel() |
| 174 | |
| 175 | retcode = process.poll() |
| 176 | if retcode: |
| 177 | if retcode == -9: # killed, assume due to timeout callback |
| 178 | raise TimeoutError(command, output='\n'.join([output, error])) |
| 179 | elif ignore != 'all' and retcode not in ignore: |
| 180 | raise subprocess.CalledProcessError(retcode, command, output='\n'.join([output, error])) |
| 181 | return output, error |
| 182 | |
| 183 | |
| 184 | def walk_modules(path): |
| 185 | """ |
| 186 | Given package name, return a list of all modules (including submodules, etc) |
| 187 | in that package. |
| 188 | |
Sergei Trofimov | 28891a8 | 2017-02-08 11:21:06 +0000 | [diff] [blame] | 189 | :raises HostError: if an exception is raised while trying to import one of the |
| 190 | modules under ``path``. The exception will have addtional |
| 191 | attributes set: ``module`` will be set to the qualified name |
| 192 | of the originating module, and ``orig_exc`` will contain |
| 193 | the original exception. |
| 194 | |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 195 | """ |
Sergei Trofimov | 28891a8 | 2017-02-08 11:21:06 +0000 | [diff] [blame] | 196 | |
| 197 | def __try_import(path): |
| 198 | try: |
| 199 | return __import__(path, {}, {}, ['']) |
| 200 | except Exception as e: |
| 201 | he = HostError('Could not load {}: {}'.format(path, str(e))) |
| 202 | he.module = path |
Sergei Trofimov | 1dd6950 | 2017-02-23 09:13:05 +0000 | [diff] [blame] | 203 | he.exc_info = sys.exc_info() |
Sergei Trofimov | 28891a8 | 2017-02-08 11:21:06 +0000 | [diff] [blame] | 204 | he.orig_exc = e |
| 205 | raise he |
| 206 | |
| 207 | root_mod = __try_import(path) |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 208 | mods = [root_mod] |
Sergei Trofimov | fef7c16 | 2017-02-23 13:15:27 +0000 | [diff] [blame] | 209 | if not hasattr(root_mod, '__path__'): |
| 210 | # root is a module not a package -- nothing to walk |
| 211 | return mods |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 212 | for _, name, ispkg in pkgutil.iter_modules(root_mod.__path__): |
| 213 | submod_path = '.'.join([path, name]) |
| 214 | if ispkg: |
| 215 | mods.extend(walk_modules(submod_path)) |
| 216 | else: |
Sergei Trofimov | 28891a8 | 2017-02-08 11:21:06 +0000 | [diff] [blame] | 217 | submod = __try_import(submod_path) |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 218 | mods.append(submod) |
| 219 | return mods |
| 220 | |
| 221 | |
| 222 | def ensure_directory_exists(dirpath): |
| 223 | """A filter for directory paths to ensure they exist.""" |
| 224 | if not os.path.isdir(dirpath): |
| 225 | os.makedirs(dirpath) |
| 226 | return dirpath |
| 227 | |
| 228 | |
| 229 | def ensure_file_directory_exists(filepath): |
| 230 | """ |
| 231 | A filter for file paths to ensure the directory of the |
| 232 | file exists and the file can be created there. The file |
| 233 | itself is *not* going to be created if it doesn't already |
| 234 | exist. |
| 235 | |
| 236 | """ |
| 237 | ensure_directory_exists(os.path.dirname(filepath)) |
| 238 | return filepath |
| 239 | |
| 240 | |
| 241 | def merge_dicts(*args, **kwargs): |
| 242 | if not len(args) >= 2: |
| 243 | raise ValueError('Must specify at least two dicts to merge.') |
| 244 | func = partial(_merge_two_dicts, **kwargs) |
| 245 | return reduce(func, args) |
| 246 | |
| 247 | |
| 248 | def _merge_two_dicts(base, other, list_duplicates='all', match_types=False, # pylint: disable=R0912,R0914 |
| 249 | dict_type=dict, should_normalize=True, should_merge_lists=True): |
| 250 | """Merge dicts normalizing their keys.""" |
| 251 | merged = dict_type() |
| 252 | base_keys = base.keys() |
| 253 | other_keys = other.keys() |
| 254 | norm = normalize if should_normalize else lambda x, y: x |
| 255 | |
| 256 | base_only = [] |
| 257 | other_only = [] |
| 258 | both = [] |
| 259 | union = [] |
| 260 | for k in base_keys: |
| 261 | if k in other_keys: |
| 262 | both.append(k) |
| 263 | else: |
| 264 | base_only.append(k) |
| 265 | union.append(k) |
| 266 | for k in other_keys: |
| 267 | if k in base_keys: |
| 268 | union.append(k) |
| 269 | else: |
| 270 | union.append(k) |
| 271 | other_only.append(k) |
| 272 | |
| 273 | for k in union: |
| 274 | if k in base_only: |
| 275 | merged[k] = norm(base[k], dict_type) |
| 276 | elif k in other_only: |
| 277 | merged[k] = norm(other[k], dict_type) |
| 278 | elif k in both: |
| 279 | base_value = base[k] |
| 280 | other_value = other[k] |
| 281 | base_type = type(base_value) |
| 282 | other_type = type(other_value) |
| 283 | if (match_types and (base_type != other_type) and |
| 284 | (base_value is not None) and (other_value is not None)): |
| 285 | raise ValueError('Type mismatch for {} got {} ({}) and {} ({})'.format(k, base_value, base_type, |
| 286 | other_value, other_type)) |
| 287 | if isinstance(base_value, dict): |
| 288 | merged[k] = _merge_two_dicts(base_value, other_value, list_duplicates, match_types, dict_type) |
| 289 | elif isinstance(base_value, list): |
| 290 | if should_merge_lists: |
| 291 | merged[k] = _merge_two_lists(base_value, other_value, list_duplicates, dict_type) |
| 292 | else: |
| 293 | merged[k] = _merge_two_lists([], other_value, list_duplicates, dict_type) |
| 294 | |
| 295 | elif isinstance(base_value, set): |
| 296 | merged[k] = norm(base_value.union(other_value), dict_type) |
| 297 | else: |
| 298 | merged[k] = norm(other_value, dict_type) |
| 299 | else: # Should never get here |
| 300 | raise AssertionError('Unexpected merge key: {}'.format(k)) |
| 301 | |
| 302 | return merged |
| 303 | |
| 304 | |
| 305 | def merge_lists(*args, **kwargs): |
| 306 | if not len(args) >= 2: |
| 307 | raise ValueError('Must specify at least two lists to merge.') |
| 308 | func = partial(_merge_two_lists, **kwargs) |
| 309 | return reduce(func, args) |
| 310 | |
| 311 | |
| 312 | def _merge_two_lists(base, other, duplicates='all', dict_type=dict): # pylint: disable=R0912 |
| 313 | """ |
| 314 | Merge lists, normalizing their entries. |
| 315 | |
| 316 | parameters: |
| 317 | |
| 318 | :base, other: the two lists to be merged. ``other`` will be merged on |
| 319 | top of base. |
| 320 | :duplicates: Indicates the strategy of handling entries that appear |
| 321 | in both lists. ``all`` will keep occurrences from both |
| 322 | lists; ``first`` will only keep occurrences from |
| 323 | ``base``; ``last`` will only keep occurrences from |
| 324 | ``other``; |
| 325 | |
| 326 | .. note:: duplicate entries that appear in the *same* list |
| 327 | will never be removed. |
| 328 | |
| 329 | """ |
| 330 | if not isiterable(base): |
| 331 | base = [base] |
| 332 | if not isiterable(other): |
| 333 | other = [other] |
| 334 | if duplicates == 'all': |
| 335 | merged_list = [] |
| 336 | for v in normalize(base, dict_type) + normalize(other, dict_type): |
| 337 | if not _check_remove_item(merged_list, v): |
| 338 | merged_list.append(v) |
| 339 | return merged_list |
| 340 | elif duplicates == 'first': |
| 341 | base_norm = normalize(base, dict_type) |
| 342 | merged_list = normalize(base, dict_type) |
| 343 | for v in base_norm: |
| 344 | _check_remove_item(merged_list, v) |
| 345 | for v in normalize(other, dict_type): |
| 346 | if not _check_remove_item(merged_list, v): |
| 347 | if v not in base_norm: |
| 348 | merged_list.append(v) # pylint: disable=no-member |
| 349 | return merged_list |
| 350 | elif duplicates == 'last': |
| 351 | other_norm = normalize(other, dict_type) |
| 352 | merged_list = [] |
| 353 | for v in normalize(base, dict_type): |
| 354 | if not _check_remove_item(merged_list, v): |
| 355 | if v not in other_norm: |
| 356 | merged_list.append(v) |
| 357 | for v in other_norm: |
| 358 | if not _check_remove_item(merged_list, v): |
| 359 | merged_list.append(v) |
| 360 | return merged_list |
| 361 | else: |
| 362 | raise ValueError('Unexpected value for list duplicates argument: {}. '.format(duplicates) + |
| 363 | 'Must be in {"all", "first", "last"}.') |
| 364 | |
| 365 | |
| 366 | def _check_remove_item(the_list, item): |
| 367 | """Helper function for merge_lists that implements checking wether an items |
| 368 | should be removed from the list and doing so if needed. Returns ``True`` if |
| 369 | the item has been removed and ``False`` otherwise.""" |
| 370 | if not isinstance(item, basestring): |
| 371 | return False |
| 372 | if not item.startswith('~'): |
| 373 | return False |
| 374 | actual_item = item[1:] |
| 375 | if actual_item in the_list: |
| 376 | del the_list[the_list.index(actual_item)] |
| 377 | return True |
| 378 | |
| 379 | |
| 380 | def normalize(value, dict_type=dict): |
| 381 | """Normalize values. Recursively normalizes dict keys to be lower case, |
| 382 | no surrounding whitespace, underscore-delimited strings.""" |
| 383 | if isinstance(value, dict): |
| 384 | normalized = dict_type() |
| 385 | for k, v in value.iteritems(): |
| 386 | key = k.strip().lower().replace(' ', '_') |
| 387 | normalized[key] = normalize(v, dict_type) |
| 388 | return normalized |
| 389 | elif isinstance(value, list): |
| 390 | return [normalize(v, dict_type) for v in value] |
| 391 | elif isinstance(value, tuple): |
| 392 | return tuple([normalize(v, dict_type) for v in value]) |
| 393 | else: |
| 394 | return value |
| 395 | |
| 396 | |
| 397 | def convert_new_lines(text): |
| 398 | """ Convert new lines to a common format. """ |
| 399 | return text.replace('\r\n', '\n').replace('\r', '\n') |
| 400 | |
| 401 | |
| 402 | def escape_quotes(text): |
| 403 | """Escape quotes, and escaped quotes, in the specified text.""" |
| 404 | return re.sub(r'\\("|\')', r'\\\\\1', text).replace('\'', '\\\'').replace('\"', '\\\"') |
| 405 | |
| 406 | |
| 407 | def escape_single_quotes(text): |
| 408 | """Escape single quotes, and escaped single quotes, in the specified text.""" |
| 409 | return re.sub(r'\\("|\')', r'\\\\\1', text).replace('\'', '\'\\\'\'') |
| 410 | |
| 411 | |
| 412 | def escape_double_quotes(text): |
| 413 | """Escape double quotes, and escaped double quotes, in the specified text.""" |
| 414 | return re.sub(r'\\("|\')', r'\\\\\1', text).replace('\"', '\\\"') |
| 415 | |
| 416 | |
| 417 | def getch(count=1): |
| 418 | """Read ``count`` characters from standard input.""" |
| 419 | if os.name == 'nt': |
| 420 | import msvcrt # pylint: disable=F0401 |
| 421 | return ''.join([msvcrt.getch() for _ in xrange(count)]) |
| 422 | else: # assume Unix |
| 423 | import tty # NOQA |
| 424 | import termios # NOQA |
| 425 | fd = sys.stdin.fileno() |
| 426 | old_settings = termios.tcgetattr(fd) |
| 427 | try: |
| 428 | tty.setraw(sys.stdin.fileno()) |
| 429 | ch = sys.stdin.read(count) |
| 430 | finally: |
| 431 | termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) |
| 432 | return ch |
| 433 | |
| 434 | |
| 435 | def isiterable(obj): |
| 436 | """Returns ``True`` if the specified object is iterable and |
| 437 | *is not a string type*, ``False`` otherwise.""" |
| 438 | return hasattr(obj, '__iter__') and not isinstance(obj, basestring) |
| 439 | |
| 440 | |
| 441 | def as_relative(path): |
| 442 | """Convert path to relative by stripping away the leading '/' on UNIX or |
| 443 | the equivant on other platforms.""" |
| 444 | path = os.path.splitdrive(path)[1] |
| 445 | return path.lstrip(os.sep) |
| 446 | |
| 447 | |
| 448 | def get_cpu_mask(cores): |
| 449 | """Return a string with the hex for the cpu mask for the specified core numbers.""" |
| 450 | mask = 0 |
| 451 | for i in cores: |
| 452 | mask |= 1 << i |
| 453 | return '0x{0:x}'.format(mask) |
| 454 | |
| 455 | |
| 456 | def which(name): |
| 457 | """Platform-independent version of UNIX which utility.""" |
| 458 | if os.name == 'nt': |
| 459 | paths = os.getenv('PATH').split(os.pathsep) |
| 460 | exts = os.getenv('PATHEXT').split(os.pathsep) |
| 461 | for path in paths: |
| 462 | testpath = os.path.join(path, name) |
| 463 | if os.path.isfile(testpath): |
| 464 | return testpath |
| 465 | for ext in exts: |
| 466 | testpathext = testpath + ext |
| 467 | if os.path.isfile(testpathext): |
| 468 | return testpathext |
| 469 | return None |
| 470 | else: # assume UNIX-like |
| 471 | try: |
| 472 | return check_output(['which', name])[0].strip() # pylint: disable=E1103 |
| 473 | except subprocess.CalledProcessError: |
| 474 | return None |
| 475 | |
| 476 | |
Sergei Trofimov | 661ba19 | 2017-10-06 16:17:56 +0100 | [diff] [blame] | 477 | # This matches most ANSI escape sequences, not just colors |
| 478 | _bash_color_regex = re.compile(r'\x1b\[[0-9;]*[a-zA-Z]') |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 479 | |
| 480 | def strip_bash_colors(text): |
| 481 | return _bash_color_regex.sub('', text) |
| 482 | |
| 483 | |
| 484 | def get_random_string(length): |
| 485 | """Returns a random ASCII string of the specified length).""" |
| 486 | return ''.join(random.choice(string.ascii_letters + string.digits) for _ in xrange(length)) |
| 487 | |
| 488 | |
| 489 | class LoadSyntaxError(Exception): |
| 490 | |
| 491 | def __init__(self, message, filepath, lineno): |
| 492 | super(LoadSyntaxError, self).__init__(message) |
| 493 | self.filepath = filepath |
| 494 | self.lineno = lineno |
| 495 | |
| 496 | def __str__(self): |
| 497 | message = 'Syntax Error in {}, line {}:\n\t{}' |
| 498 | return message.format(self.filepath, self.lineno, self.message) |
| 499 | |
| 500 | |
| 501 | RAND_MOD_NAME_LEN = 30 |
| 502 | BAD_CHARS = string.punctuation + string.whitespace |
| 503 | TRANS_TABLE = string.maketrans(BAD_CHARS, '_' * len(BAD_CHARS)) |
| 504 | |
| 505 | |
| 506 | def to_identifier(text): |
| 507 | """Converts text to a valid Python identifier by replacing all |
| 508 | whitespace and punctuation.""" |
| 509 | return re.sub('_+', '_', text.translate(TRANS_TABLE)) |
| 510 | |
| 511 | |
| 512 | def unique(alist): |
| 513 | """ |
| 514 | Returns a list containing only unique elements from the input list (but preserves |
| 515 | order, unlike sets). |
| 516 | |
| 517 | """ |
| 518 | result = [] |
| 519 | for item in alist: |
| 520 | if item not in result: |
| 521 | result.append(item) |
| 522 | return result |
| 523 | |
| 524 | |
| 525 | def ranges_to_list(ranges_string): |
| 526 | """Converts a sysfs-style ranges string, e.g. ``"0,2-4"``, into a list ,e.g ``[0,2,3,4]``""" |
| 527 | values = [] |
| 528 | for rg in ranges_string.split(','): |
| 529 | if '-' in rg: |
| 530 | first, last = map(int, rg.split('-')) |
| 531 | values.extend(xrange(first, last + 1)) |
| 532 | else: |
| 533 | values.append(int(rg)) |
| 534 | return values |
| 535 | |
| 536 | |
| 537 | def list_to_ranges(values): |
| 538 | """Converts a list, e.g ``[0,2,3,4]``, into a sysfs-style ranges string, e.g. ``"0,2-4"``""" |
| 539 | range_groups = [] |
| 540 | for _, g in groupby(enumerate(values), lambda (i, x): i - x): |
| 541 | range_groups.append(map(itemgetter(1), g)) |
| 542 | range_strings = [] |
| 543 | for group in range_groups: |
| 544 | if len(group) == 1: |
| 545 | range_strings.append(str(group[0])) |
| 546 | else: |
| 547 | range_strings.append('{}-{}'.format(group[0], group[-1])) |
| 548 | return ','.join(range_strings) |
| 549 | |
| 550 | |
| 551 | def list_to_mask(values, base=0x0): |
| 552 | """Converts the specified list of integer values into |
| 553 | a bit mask for those values. Optinally, the list can be |
| 554 | applied to an existing mask.""" |
| 555 | for v in values: |
| 556 | base |= (1 << v) |
| 557 | return base |
| 558 | |
| 559 | |
| 560 | def mask_to_list(mask): |
| 561 | """Converts the specfied integer bitmask into a list of |
| 562 | indexes of bits that are set in the mask.""" |
| 563 | size = len(bin(mask)) - 2 # because of "0b" |
| 564 | return [size - i - 1 for i in xrange(size) |
| 565 | if mask & (1 << size - i - 1)] |
| 566 | |
| 567 | |
| 568 | __memo_cache = {} |
| 569 | |
| 570 | |
Sergei Trofimov | d7aac2b | 2016-09-02 13:22:09 +0100 | [diff] [blame] | 571 | def reset_memo_cache(): |
| 572 | __memo_cache.clear() |
| 573 | |
| 574 | |
Sergei Trofimov | 6d854fd | 2016-09-06 09:57:58 +0100 | [diff] [blame] | 575 | def __get_memo_id(obj): |
| 576 | """ |
| 577 | An object's id() may be re-used after an object is freed, so it's not |
| 578 | sufficiently unique to identify params for the memo cache (two different |
| 579 | params may end up with the same id). this attempts to generate a more unique |
| 580 | ID string. |
| 581 | """ |
| 582 | obj_id = id(obj) |
Sergei Trofimov | cae239d | 2016-10-06 08:44:42 +0100 | [diff] [blame] | 583 | try: |
Sergei Trofimov | 09ec88e | 2016-10-04 17:57:46 +0100 | [diff] [blame] | 584 | return '{}/{}'.format(obj_id, hash(obj)) |
Sergei Trofimov | cae239d | 2016-10-06 08:44:42 +0100 | [diff] [blame] | 585 | except TypeError: # obj is not hashable |
Sergei Trofimov | 09ec88e | 2016-10-04 17:57:46 +0100 | [diff] [blame] | 586 | obj_pyobj = ctypes.cast(obj_id, ctypes.py_object) |
| 587 | # TODO: Note: there is still a possibility of a clash here. If Two |
| 588 | # different objects get assigned the same ID, an are large and are |
| 589 | # identical in the first thirty two bytes. This shouldn't be much of an |
| 590 | # issue in the current application of memoizing Target calls, as it's very |
| 591 | # unlikely that a target will get passed large params; but may cause |
| 592 | # problems in other applications, e.g. when memoizing results of operations |
| 593 | # on large arrays. I can't really think of a good way around that apart |
| 594 | # form, e.g., md5 hashing the entire raw object, which will have an |
| 595 | # undesirable impact on performance. |
| 596 | num_bytes = min(ctypes.sizeof(obj_pyobj), 32) |
| 597 | obj_bytes = ctypes.string_at(ctypes.addressof(obj_pyobj), num_bytes) |
| 598 | return '{}/{}'.format(obj_id, obj_bytes) |
Sergei Trofimov | 6d854fd | 2016-09-06 09:57:58 +0100 | [diff] [blame] | 599 | |
| 600 | |
Michele Di Giorgio | 539e9b3 | 2016-06-22 17:54:59 +0100 | [diff] [blame] | 601 | @wrapt.decorator |
| 602 | def memoized(wrapped, instance, args, kwargs): |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 603 | """A decorator for memoizing functions and methods.""" |
Michele Di Giorgio | 539e9b3 | 2016-06-22 17:54:59 +0100 | [diff] [blame] | 604 | func_id = repr(wrapped) |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 605 | |
| 606 | def memoize_wrapper(*args, **kwargs): |
Sergei Trofimov | 6d854fd | 2016-09-06 09:57:58 +0100 | [diff] [blame] | 607 | id_string = func_id + ','.join([__get_memo_id(a) for a in args]) |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 608 | id_string += ','.join('{}={}'.format(k, v) |
| 609 | for k, v in kwargs.iteritems()) |
| 610 | if id_string not in __memo_cache: |
Michele Di Giorgio | 539e9b3 | 2016-06-22 17:54:59 +0100 | [diff] [blame] | 611 | __memo_cache[id_string] = wrapped(*args, **kwargs) |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 612 | return __memo_cache[id_string] |
| 613 | |
Michele Di Giorgio | 539e9b3 | 2016-06-22 17:54:59 +0100 | [diff] [blame] | 614 | return memoize_wrapper(*args, **kwargs) |
Sergei Trofimov | 4e6afe9 | 2015-10-09 09:30:04 +0100 | [diff] [blame] | 615 | |