Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1 | # Copyright 2017 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
mbligh | aece77e | 2009-01-21 19:06:52 +0000 | [diff] [blame] | 5 | """ |
| 6 | Convenience functions for use by tests or whomever. |
mbligh | aece77e | 2009-01-21 19:06:52 +0000 | [diff] [blame] | 7 | """ |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 8 | |
| 9 | # pylint: disable=missing-docstring |
| 10 | |
Kuo-Hsin Yang | 9cd699c | 2018-03-23 12:13:27 +0800 | [diff] [blame^] | 11 | import collections |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 12 | import commands |
| 13 | import fnmatch |
| 14 | import glob |
| 15 | import json |
| 16 | import logging |
| 17 | import math |
| 18 | import multiprocessing |
mbligh | cc03864 | 2009-02-05 20:15:52 +0000 | [diff] [blame] | 19 | import os |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 20 | import pickle |
| 21 | import platform |
| 22 | import re |
| 23 | import shutil |
| 24 | import signal |
| 25 | import tempfile |
| 26 | import time |
| 27 | import uuid |
| 28 | |
| 29 | from autotest_lib.client.common_lib import error |
| 30 | from autotest_lib.client.common_lib import magic |
| 31 | from autotest_lib.client.common_lib import utils |
mbligh | aece77e | 2009-01-21 19:06:52 +0000 | [diff] [blame] | 32 | |
| 33 | from autotest_lib.client.common_lib.utils import * |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 34 | |
| 35 | |
| 36 | def grep(pattern, file): |
| 37 | """ |
| 38 | This is mainly to fix the return code inversion from grep |
| 39 | Also handles compressed files. |
| 40 | |
| 41 | returns 1 if the pattern is present in the file, 0 if not. |
| 42 | """ |
| 43 | command = 'grep "%s" > /dev/null' % pattern |
| 44 | ret = cat_file_to_cmd(file, command, ignore_status=True) |
| 45 | return not ret |
| 46 | |
| 47 | |
| 48 | def difflist(list1, list2): |
| 49 | """returns items in list2 that are not in list1""" |
| 50 | diff = []; |
| 51 | for x in list2: |
| 52 | if x not in list1: |
| 53 | diff.append(x) |
| 54 | return diff |
| 55 | |
| 56 | |
| 57 | def cat_file_to_cmd(file, command, ignore_status=0, return_output=False): |
| 58 | """ |
| 59 | equivalent to 'cat file | command' but knows to use |
| 60 | zcat or bzcat if appropriate |
| 61 | """ |
| 62 | if not os.path.isfile(file): |
| 63 | raise NameError('invalid file %s to cat to command %s' |
| 64 | % (file, command)) |
| 65 | |
| 66 | if return_output: |
| 67 | run_cmd = utils.system_output |
| 68 | else: |
| 69 | run_cmd = utils.system |
| 70 | |
| 71 | if magic.guess_type(file) == 'application/x-bzip2': |
| 72 | cat = 'bzcat' |
| 73 | elif magic.guess_type(file) == 'application/x-gzip': |
| 74 | cat = 'zcat' |
| 75 | else: |
| 76 | cat = 'cat' |
| 77 | return run_cmd('%s %s | %s' % (cat, file, command), |
| 78 | ignore_status=ignore_status) |
| 79 | |
| 80 | |
| 81 | def extract_tarball_to_dir(tarball, dir): |
| 82 | """ |
| 83 | Extract a tarball to a specified directory name instead of whatever |
| 84 | the top level of a tarball is - useful for versioned directory names, etc |
| 85 | """ |
| 86 | if os.path.exists(dir): |
| 87 | if os.path.isdir(dir): |
| 88 | shutil.rmtree(dir) |
| 89 | else: |
| 90 | os.remove(dir) |
| 91 | pwd = os.getcwd() |
| 92 | os.chdir(os.path.dirname(os.path.abspath(dir))) |
| 93 | newdir = extract_tarball(tarball) |
| 94 | os.rename(newdir, dir) |
| 95 | os.chdir(pwd) |
| 96 | |
| 97 | |
| 98 | def extract_tarball(tarball): |
| 99 | """Returns the directory extracted by the tarball.""" |
| 100 | extracted = cat_file_to_cmd(tarball, 'tar xvf - 2>/dev/null', |
| 101 | return_output=True).splitlines() |
| 102 | |
| 103 | dir = None |
| 104 | |
| 105 | for line in extracted: |
| 106 | if line.startswith('./'): |
| 107 | line = line[2:] |
| 108 | if not line or line == '.': |
| 109 | continue |
| 110 | topdir = line.split('/')[0] |
| 111 | if os.path.isdir(topdir): |
| 112 | if dir: |
| 113 | assert(dir == topdir) |
| 114 | else: |
| 115 | dir = topdir |
| 116 | if dir: |
| 117 | return dir |
| 118 | else: |
| 119 | raise NameError('extracting tarball produced no dir') |
| 120 | |
| 121 | |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 122 | def force_copy(src, dest): |
| 123 | """Replace dest with a new copy of src, even if it exists""" |
| 124 | if os.path.isfile(dest): |
| 125 | os.remove(dest) |
| 126 | if os.path.isdir(dest): |
| 127 | dest = os.path.join(dest, os.path.basename(src)) |
| 128 | shutil.copyfile(src, dest) |
| 129 | return dest |
| 130 | |
| 131 | |
| 132 | def force_link(src, dest): |
| 133 | """Link src to dest, overwriting it if it exists""" |
| 134 | return utils.system("ln -sf %s %s" % (src, dest)) |
| 135 | |
| 136 | |
| 137 | def file_contains_pattern(file, pattern): |
| 138 | """Return true if file contains the specified egrep pattern""" |
| 139 | if not os.path.isfile(file): |
| 140 | raise NameError('file %s does not exist' % file) |
| 141 | return not utils.system('egrep -q "' + pattern + '" ' + file, |
| 142 | ignore_status=True) |
| 143 | |
| 144 | |
| 145 | def list_grep(list, pattern): |
| 146 | """True if any item in list matches the specified pattern.""" |
| 147 | compiled = re.compile(pattern) |
| 148 | for line in list: |
| 149 | match = compiled.search(line) |
| 150 | if (match): |
| 151 | return 1 |
| 152 | return 0 |
| 153 | |
| 154 | |
| 155 | def get_os_vendor(): |
| 156 | """Try to guess what's the os vendor |
| 157 | """ |
| 158 | if os.path.isfile('/etc/SuSE-release'): |
| 159 | return 'SUSE' |
| 160 | |
| 161 | issue = '/etc/issue' |
| 162 | |
| 163 | if not os.path.isfile(issue): |
| 164 | return 'Unknown' |
| 165 | |
| 166 | if file_contains_pattern(issue, 'Red Hat'): |
| 167 | return 'Red Hat' |
| 168 | elif file_contains_pattern(issue, 'Fedora'): |
| 169 | return 'Fedora Core' |
| 170 | elif file_contains_pattern(issue, 'SUSE'): |
| 171 | return 'SUSE' |
| 172 | elif file_contains_pattern(issue, 'Ubuntu'): |
| 173 | return 'Ubuntu' |
| 174 | elif file_contains_pattern(issue, 'Debian'): |
| 175 | return 'Debian' |
| 176 | else: |
| 177 | return 'Unknown' |
| 178 | |
| 179 | |
| 180 | def get_cc(): |
| 181 | try: |
| 182 | return os.environ['CC'] |
| 183 | except KeyError: |
| 184 | return 'gcc' |
| 185 | |
| 186 | |
| 187 | def get_vmlinux(): |
| 188 | """Return the full path to vmlinux |
| 189 | |
| 190 | Ahem. This is crap. Pray harder. Bad Martin. |
| 191 | """ |
| 192 | vmlinux = '/boot/vmlinux-%s' % utils.system_output('uname -r') |
| 193 | if os.path.isfile(vmlinux): |
| 194 | return vmlinux |
| 195 | vmlinux = '/lib/modules/%s/build/vmlinux' % utils.system_output('uname -r') |
| 196 | if os.path.isfile(vmlinux): |
| 197 | return vmlinux |
| 198 | return None |
| 199 | |
| 200 | |
| 201 | def get_systemmap(): |
| 202 | """Return the full path to System.map |
| 203 | |
| 204 | Ahem. This is crap. Pray harder. Bad Martin. |
| 205 | """ |
| 206 | map = '/boot/System.map-%s' % utils.system_output('uname -r') |
| 207 | if os.path.isfile(map): |
| 208 | return map |
| 209 | map = '/lib/modules/%s/build/System.map' % utils.system_output('uname -r') |
| 210 | if os.path.isfile(map): |
| 211 | return map |
| 212 | return None |
| 213 | |
| 214 | |
| 215 | def get_modules_dir(): |
| 216 | """Return the modules dir for the running kernel version""" |
| 217 | kernel_version = utils.system_output('uname -r') |
| 218 | return '/lib/modules/%s/kernel' % kernel_version |
| 219 | |
| 220 | |
| 221 | _CPUINFO_RE = re.compile(r'^(?P<key>[^\t]*)\t*: ?(?P<value>.*)$') |
| 222 | |
| 223 | |
| 224 | def get_cpuinfo(): |
| 225 | """Read /proc/cpuinfo and convert to a list of dicts.""" |
| 226 | cpuinfo = [] |
| 227 | with open('/proc/cpuinfo', 'r') as f: |
| 228 | cpu = {} |
| 229 | for line in f: |
| 230 | line = line.strip() |
| 231 | if not line: |
| 232 | cpuinfo.append(cpu) |
| 233 | cpu = {} |
| 234 | continue |
| 235 | match = _CPUINFO_RE.match(line) |
| 236 | cpu[match.group('key')] = match.group('value') |
| 237 | if cpu: |
| 238 | # cpuinfo usually ends in a blank line, so this shouldn't happen. |
| 239 | cpuinfo.append(cpu) |
| 240 | return cpuinfo |
| 241 | |
| 242 | |
| 243 | def get_cpu_arch(): |
| 244 | """Work out which CPU architecture we're running on""" |
| 245 | f = open('/proc/cpuinfo', 'r') |
| 246 | cpuinfo = f.readlines() |
| 247 | f.close() |
| 248 | if list_grep(cpuinfo, '^cpu.*(RS64|POWER3|Broadband Engine)'): |
| 249 | return 'power' |
| 250 | elif list_grep(cpuinfo, '^cpu.*POWER4'): |
| 251 | return 'power4' |
| 252 | elif list_grep(cpuinfo, '^cpu.*POWER5'): |
| 253 | return 'power5' |
| 254 | elif list_grep(cpuinfo, '^cpu.*POWER6'): |
| 255 | return 'power6' |
| 256 | elif list_grep(cpuinfo, '^cpu.*POWER7'): |
| 257 | return 'power7' |
| 258 | elif list_grep(cpuinfo, '^cpu.*PPC970'): |
| 259 | return 'power970' |
| 260 | elif list_grep(cpuinfo, 'ARM'): |
| 261 | return 'arm' |
| 262 | elif list_grep(cpuinfo, '^flags.*:.* lm .*'): |
| 263 | return 'x86_64' |
| 264 | elif list_grep(cpuinfo, 'CPU.*implementer.*0x41'): |
| 265 | return 'arm' |
| 266 | else: |
| 267 | return 'i386' |
| 268 | |
| 269 | |
| 270 | def get_arm_soc_family_from_devicetree(): |
| 271 | """ |
| 272 | Work out which ARM SoC we're running on based on the 'compatible' property |
| 273 | of the base node of devicetree, if it exists. |
| 274 | """ |
| 275 | devicetree_compatible = '/sys/firmware/devicetree/base/compatible' |
| 276 | if not os.path.isfile(devicetree_compatible): |
| 277 | return None |
| 278 | f = open(devicetree_compatible, 'r') |
| 279 | compatible = f.readlines() |
| 280 | f.close() |
| 281 | if list_grep(compatible, 'rk3399'): |
| 282 | return 'rockchip' |
| 283 | elif list_grep(compatible, 'mt8173'): |
| 284 | return 'mediatek' |
| 285 | return None |
| 286 | |
| 287 | |
| 288 | def get_arm_soc_family(): |
| 289 | """Work out which ARM SoC we're running on""" |
| 290 | family = get_arm_soc_family_from_devicetree() |
| 291 | if family is not None: |
| 292 | return family |
| 293 | |
| 294 | f = open('/proc/cpuinfo', 'r') |
| 295 | cpuinfo = f.readlines() |
| 296 | f.close() |
| 297 | if list_grep(cpuinfo, 'EXYNOS5'): |
| 298 | return 'exynos5' |
| 299 | elif list_grep(cpuinfo, 'Tegra'): |
| 300 | return 'tegra' |
| 301 | elif list_grep(cpuinfo, 'Rockchip'): |
| 302 | return 'rockchip' |
| 303 | return 'arm' |
| 304 | |
| 305 | |
| 306 | def get_cpu_soc_family(): |
| 307 | """Like get_cpu_arch, but for ARM, returns the SoC family name""" |
| 308 | f = open('/proc/cpuinfo', 'r') |
| 309 | cpuinfo = f.readlines() |
| 310 | f.close() |
| 311 | family = get_cpu_arch() |
| 312 | if family == 'arm': |
| 313 | family = get_arm_soc_family() |
| 314 | if list_grep(cpuinfo, '^vendor_id.*:.*AMD'): |
| 315 | family = 'amd' |
| 316 | return family |
| 317 | |
| 318 | |
| 319 | INTEL_UARCH_TABLE = { |
Harry Pan | c9897aa | 2017-07-31 20:17:52 +0800 | [diff] [blame] | 320 | '06_4C': 'Airmont', |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 321 | '06_1C': 'Atom', |
| 322 | '06_26': 'Atom', |
Harry Pan | c9897aa | 2017-07-31 20:17:52 +0800 | [diff] [blame] | 323 | '06_27': 'Atom', |
| 324 | '06_35': 'Atom', |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 325 | '06_36': 'Atom', |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 326 | '06_3D': 'Broadwell', |
Harry Pan | c9897aa | 2017-07-31 20:17:52 +0800 | [diff] [blame] | 327 | '06_47': 'Broadwell', |
Harry Pan | 45cb0e0 | 2017-08-22 22:19:59 +0800 | [diff] [blame] | 328 | '06_4F': 'Broadwell', |
| 329 | '06_56': 'Broadwell', |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 330 | '06_0D': 'Dothan', |
Harry Pan | c9897aa | 2017-07-31 20:17:52 +0800 | [diff] [blame] | 331 | '06_5C': 'Goldmont', |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 332 | '06_3C': 'Haswell', |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 333 | '06_45': 'Haswell', |
| 334 | '06_46': 'Haswell', |
Harry Pan | c9897aa | 2017-07-31 20:17:52 +0800 | [diff] [blame] | 335 | '06_3F': 'Haswell-E', |
| 336 | '06_3A': 'Ivy Bridge', |
| 337 | '06_3E': 'Ivy Bridge-E', |
| 338 | '06_8E': 'Kaby Lake', |
| 339 | '06_9E': 'Kaby Lake', |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 340 | '06_0F': 'Merom', |
| 341 | '06_16': 'Merom', |
| 342 | '06_17': 'Nehalem', |
| 343 | '06_1A': 'Nehalem', |
| 344 | '06_1D': 'Nehalem', |
| 345 | '06_1E': 'Nehalem', |
| 346 | '06_1F': 'Nehalem', |
| 347 | '06_2E': 'Nehalem', |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 348 | '0F_03': 'Prescott', |
| 349 | '0F_04': 'Prescott', |
| 350 | '0F_06': 'Presler', |
Harry Pan | c9897aa | 2017-07-31 20:17:52 +0800 | [diff] [blame] | 351 | '06_2A': 'Sandy Bridge', |
| 352 | '06_2D': 'Sandy Bridge', |
| 353 | '06_37': 'Silvermont', |
| 354 | '06_4A': 'Silvermont', |
| 355 | '06_4D': 'Silvermont', |
| 356 | '06_5A': 'Silvermont', |
| 357 | '06_5D': 'Silvermont', |
| 358 | '06_4E': 'Skylake', |
| 359 | '06_5E': 'Skylake', |
| 360 | '06_55': 'Skylake', |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 361 | '06_25': 'Westmere', |
| 362 | '06_2C': 'Westmere', |
| 363 | '06_2F': 'Westmere', |
| 364 | } |
| 365 | |
| 366 | |
| 367 | def get_intel_cpu_uarch(numeric=False): |
| 368 | """Return the Intel microarchitecture we're running on, or None. |
| 369 | |
| 370 | Returns None if this is not an Intel CPU. Returns the family and model as |
| 371 | underscore-separated hex (per Intel manual convention) if the uarch is not |
| 372 | known, or if numeric is True. |
| 373 | """ |
| 374 | if not get_current_kernel_arch().startswith('x86'): |
| 375 | return None |
| 376 | cpuinfo = get_cpuinfo()[0] |
| 377 | if cpuinfo['vendor_id'] != 'GenuineIntel': |
| 378 | return None |
| 379 | family_model = '%02X_%02X' % (int(cpuinfo['cpu family']), |
| 380 | int(cpuinfo['model'])) |
| 381 | if numeric: |
| 382 | return family_model |
| 383 | return INTEL_UARCH_TABLE.get(family_model, family_model) |
| 384 | |
| 385 | |
| 386 | def get_current_kernel_arch(): |
| 387 | """Get the machine architecture, now just a wrap of 'uname -m'.""" |
| 388 | return os.popen('uname -m').read().rstrip() |
| 389 | |
| 390 | |
| 391 | def get_file_arch(filename): |
| 392 | # -L means follow symlinks |
| 393 | file_data = utils.system_output('file -L ' + filename) |
| 394 | if file_data.count('80386'): |
| 395 | return 'i386' |
| 396 | return None |
| 397 | |
| 398 | |
| 399 | def count_cpus(): |
| 400 | """number of CPUs in the local machine according to /proc/cpuinfo""" |
| 401 | try: |
| 402 | return multiprocessing.cpu_count() |
| 403 | except Exception: |
| 404 | logging.exception('can not get cpu count from' |
| 405 | ' multiprocessing.cpu_count()') |
| 406 | cpuinfo = get_cpuinfo() |
| 407 | # Returns at least one cpu. Check comment #1 in crosbug.com/p/9582. |
| 408 | return len(cpuinfo) or 1 |
| 409 | |
| 410 | |
| 411 | def cpu_online_map(): |
| 412 | """ |
| 413 | Check out the available cpu online map |
| 414 | """ |
| 415 | cpuinfo = get_cpuinfo() |
| 416 | cpus = [] |
| 417 | for cpu in cpuinfo: |
| 418 | cpus.append(cpu['processor']) # grab cpu number |
| 419 | return cpus |
| 420 | |
| 421 | |
| 422 | def get_cpu_family(): |
| 423 | cpuinfo = get_cpuinfo()[0] |
| 424 | return int(cpuinfo['cpu_family']) |
| 425 | |
| 426 | |
| 427 | def get_cpu_vendor(): |
| 428 | cpuinfo = get_cpuinfo() |
| 429 | vendors = [cpu['vendor_id'] for cpu in cpuinfo] |
| 430 | for v in vendors[1:]: |
| 431 | if v != vendors[0]: |
| 432 | raise error.TestError('multiple cpu vendors found: ' + str(vendors)) |
| 433 | return vendors[0] |
| 434 | |
| 435 | |
| 436 | def probe_cpus(): |
| 437 | """ |
| 438 | This routine returns a list of cpu devices found under |
| 439 | /sys/devices/system/cpu. |
| 440 | """ |
| 441 | cmd = 'find /sys/devices/system/cpu/ -maxdepth 1 -type d -name cpu*' |
| 442 | return utils.system_output(cmd).splitlines() |
| 443 | |
| 444 | |
| 445 | # Returns total memory in kb |
| 446 | def read_from_meminfo(key): |
| 447 | meminfo = utils.system_output('grep %s /proc/meminfo' % key) |
| 448 | return int(re.search(r'\d+', meminfo).group(0)) |
| 449 | |
| 450 | |
| 451 | def memtotal(): |
| 452 | return read_from_meminfo('MemTotal') |
| 453 | |
| 454 | |
| 455 | def freememtotal(): |
| 456 | return read_from_meminfo('MemFree') |
| 457 | |
| 458 | def usable_memtotal(): |
| 459 | # Reserved 5% for OS use |
| 460 | return int(read_from_meminfo('MemFree') * 0.95) |
| 461 | |
Ben Cheng | ca353aa | 2017-11-28 17:44:10 +0800 | [diff] [blame] | 462 | def swaptotal(): |
| 463 | return read_from_meminfo('SwapTotal') |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 464 | |
| 465 | def rounded_memtotal(): |
| 466 | # Get total of all physical mem, in kbytes |
| 467 | usable_kbytes = memtotal() |
| 468 | # usable_kbytes is system's usable DRAM in kbytes, |
| 469 | # as reported by memtotal() from device /proc/meminfo memtotal |
| 470 | # after Linux deducts 1.5% to 5.1% for system table overhead |
| 471 | # Undo the unknown actual deduction by rounding up |
| 472 | # to next small multiple of a big power-of-two |
| 473 | # eg 12GB - 5.1% gets rounded back up to 12GB |
| 474 | mindeduct = 0.015 # 1.5 percent |
| 475 | maxdeduct = 0.055 # 5.5 percent |
| 476 | # deduction range 1.5% .. 5.5% supports physical mem sizes |
| 477 | # 6GB .. 12GB in steps of .5GB |
| 478 | # 12GB .. 24GB in steps of 1 GB |
| 479 | # 24GB .. 48GB in steps of 2 GB ... |
| 480 | # Finer granularity in physical mem sizes would require |
| 481 | # tighter spread between min and max possible deductions |
| 482 | |
| 483 | # increase mem size by at least min deduction, without rounding |
| 484 | min_kbytes = int(usable_kbytes / (1.0 - mindeduct)) |
| 485 | # increase mem size further by 2**n rounding, by 0..roundKb or more |
| 486 | round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes |
| 487 | # find least binary roundup 2**n that covers worst-cast roundKb |
| 488 | mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2))) |
| 489 | # have round_kbytes <= mod2n < round_kbytes*2 |
| 490 | # round min_kbytes up to next multiple of mod2n |
| 491 | phys_kbytes = min_kbytes + mod2n - 1 |
| 492 | phys_kbytes = phys_kbytes - (phys_kbytes % mod2n) # clear low bits |
| 493 | return phys_kbytes |
| 494 | |
| 495 | |
Kuo-Hsin Yang | 9cd699c | 2018-03-23 12:13:27 +0800 | [diff] [blame^] | 496 | _MEMINFO_RE = re.compile('^(\w+)(\(\w+\))?:\s+(\d+)') |
| 497 | |
| 498 | |
| 499 | def get_meminfo(): |
| 500 | """Returns a namedtuple of pairs from /proc/meminfo. |
| 501 | |
| 502 | Example /proc/meminfo snippets: |
| 503 | MemTotal: 2048000 kB |
| 504 | Active(anon): 409600 kB |
| 505 | Example usage: |
| 506 | meminfo = utils.get_meminfo() |
| 507 | print meminfo.Active_anon |
| 508 | """ |
| 509 | info = {} |
| 510 | with _open_file('/proc/meminfo') as f: |
| 511 | for line in f: |
| 512 | m = _MEMINFO_RE.match(line) |
| 513 | if m: |
| 514 | if m.group(2): |
| 515 | name = m.group(1) + '_' + m.group(2)[1:-1] |
| 516 | else: |
| 517 | name = m.group(1) |
| 518 | info[name] = int(m.group(3)) |
| 519 | return collections.namedtuple('MemInfo', info.keys())(**info) |
| 520 | |
| 521 | |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 522 | def sysctl(key, value=None): |
| 523 | """Generic implementation of sysctl, to read and write. |
| 524 | |
| 525 | @param key: A location under /proc/sys |
| 526 | @param value: If not None, a value to write into the sysctl. |
| 527 | |
| 528 | @return The single-line sysctl value as a string. |
| 529 | """ |
| 530 | path = '/proc/sys/%s' % key |
| 531 | if value is not None: |
| 532 | utils.write_one_line(path, str(value)) |
| 533 | return utils.read_one_line(path) |
| 534 | |
| 535 | |
| 536 | def sysctl_kernel(key, value=None): |
| 537 | """(Very) partial implementation of sysctl, for kernel params""" |
| 538 | if value is not None: |
| 539 | # write |
| 540 | utils.write_one_line('/proc/sys/kernel/%s' % key, str(value)) |
| 541 | else: |
| 542 | # read |
| 543 | out = utils.read_one_line('/proc/sys/kernel/%s' % key) |
| 544 | return int(re.search(r'\d+', out).group(0)) |
| 545 | |
| 546 | |
| 547 | def _convert_exit_status(sts): |
| 548 | if os.WIFSIGNALED(sts): |
| 549 | return -os.WTERMSIG(sts) |
| 550 | elif os.WIFEXITED(sts): |
| 551 | return os.WEXITSTATUS(sts) |
| 552 | else: |
| 553 | # impossible? |
| 554 | raise RuntimeError("Unknown exit status %d!" % sts) |
| 555 | |
| 556 | |
| 557 | def where_art_thy_filehandles(): |
| 558 | """Dump the current list of filehandles""" |
| 559 | os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid()) |
| 560 | |
| 561 | |
Kristoffer Erlandsson | 5877668 | 2017-11-06 11:32:49 +0100 | [diff] [blame] | 562 | def get_num_allocated_file_handles(): |
| 563 | """ |
Kristoffer Erlandsson | a692885 | 2017-11-09 10:04:31 +0100 | [diff] [blame] | 564 | Returns the number of currently allocated file handles. |
Kristoffer Erlandsson | 5877668 | 2017-11-06 11:32:49 +0100 | [diff] [blame] | 565 | |
| 566 | Gets this information by parsing /proc/sys/fs/file-nr. |
| 567 | See https://www.kernel.org/doc/Documentation/sysctl/fs.txt |
| 568 | for details on this file. |
| 569 | """ |
| 570 | with _open_file('/proc/sys/fs/file-nr') as f: |
| 571 | line = f.readline() |
| 572 | allocated_handles = int(line.split()[0]) |
| 573 | return allocated_handles |
| 574 | |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 575 | def print_to_tty(string): |
| 576 | """Output string straight to the tty""" |
| 577 | open('/dev/tty', 'w').write(string + '\n') |
| 578 | |
| 579 | |
| 580 | def dump_object(object): |
| 581 | """Dump an object's attributes and methods |
| 582 | |
| 583 | kind of like dir() |
| 584 | """ |
| 585 | for item in object.__dict__.iteritems(): |
| 586 | print item |
| 587 | try: |
| 588 | (key, value) = item |
| 589 | dump_object(value) |
| 590 | except: |
| 591 | continue |
| 592 | |
| 593 | |
| 594 | def environ(env_key): |
| 595 | """return the requested environment variable, or '' if unset""" |
| 596 | if (os.environ.has_key(env_key)): |
| 597 | return os.environ[env_key] |
| 598 | else: |
| 599 | return '' |
| 600 | |
| 601 | |
| 602 | def prepend_path(newpath, oldpath): |
| 603 | """prepend newpath to oldpath""" |
| 604 | if (oldpath): |
| 605 | return newpath + ':' + oldpath |
| 606 | else: |
| 607 | return newpath |
| 608 | |
| 609 | |
| 610 | def append_path(oldpath, newpath): |
| 611 | """append newpath to oldpath""" |
| 612 | if (oldpath): |
| 613 | return oldpath + ':' + newpath |
| 614 | else: |
| 615 | return newpath |
| 616 | |
| 617 | |
| 618 | _TIME_OUTPUT_RE = re.compile( |
| 619 | r'([\d\.]*)user ([\d\.]*)system ' |
| 620 | r'(\d*):([\d\.]*)elapsed (\d*)%CPU') |
| 621 | |
| 622 | |
| 623 | def avgtime_print(dir): |
| 624 | """ Calculate some benchmarking statistics. |
| 625 | Input is a directory containing a file called 'time'. |
| 626 | File contains one-per-line results of /usr/bin/time. |
| 627 | Output is average Elapsed, User, and System time in seconds, |
| 628 | and average CPU percentage. |
| 629 | """ |
| 630 | user = system = elapsed = cpu = count = 0 |
| 631 | with open(dir + "/time") as f: |
| 632 | for line in f: |
| 633 | try: |
| 634 | m = _TIME_OUTPUT_RE.match(line); |
| 635 | user += float(m.group(1)) |
| 636 | system += float(m.group(2)) |
| 637 | elapsed += (float(m.group(3)) * 60) + float(m.group(4)) |
| 638 | cpu += float(m.group(5)) |
| 639 | count += 1 |
| 640 | except: |
| 641 | raise ValueError("badly formatted times") |
| 642 | |
| 643 | return "Elapsed: %0.2fs User: %0.2fs System: %0.2fs CPU: %0.0f%%" % \ |
| 644 | (elapsed / count, user / count, system / count, cpu / count) |
| 645 | |
| 646 | |
| 647 | def to_seconds(time_string): |
| 648 | """Converts a string in M+:SS.SS format to S+.SS""" |
| 649 | elts = time_string.split(':') |
| 650 | if len(elts) == 1: |
| 651 | return time_string |
| 652 | return str(int(elts[0]) * 60 + float(elts[1])) |
| 653 | |
| 654 | |
| 655 | _TIME_OUTPUT_RE_2 = re.compile(r'(.*?)user (.*?)system (.*?)elapsed') |
| 656 | |
| 657 | |
| 658 | def extract_all_time_results(results_string): |
| 659 | """Extract user, system, and elapsed times into a list of tuples""" |
| 660 | results = [] |
| 661 | for result in _TIME_OUTPUT_RE_2.findall(results_string): |
| 662 | results.append(tuple([to_seconds(elt) for elt in result])) |
| 663 | return results |
| 664 | |
| 665 | |
| 666 | def running_config(): |
| 667 | """ |
| 668 | Return path of config file of the currently running kernel |
| 669 | """ |
| 670 | version = utils.system_output('uname -r') |
| 671 | for config in ('/proc/config.gz', \ |
| 672 | '/boot/config-%s' % version, |
| 673 | '/lib/modules/%s/build/.config' % version): |
| 674 | if os.path.isfile(config): |
| 675 | return config |
| 676 | return None |
| 677 | |
| 678 | |
| 679 | def check_for_kernel_feature(feature): |
| 680 | config = running_config() |
| 681 | |
| 682 | if not config: |
| 683 | raise TypeError("Can't find kernel config file") |
| 684 | |
| 685 | if magic.guess_type(config) == 'application/x-gzip': |
| 686 | grep = 'zgrep' |
| 687 | else: |
| 688 | grep = 'grep' |
| 689 | grep += ' ^CONFIG_%s= %s' % (feature, config) |
| 690 | |
| 691 | if not utils.system_output(grep, ignore_status=True): |
| 692 | raise ValueError("Kernel doesn't have a %s feature" % (feature)) |
| 693 | |
| 694 | |
| 695 | def check_glibc_ver(ver): |
| 696 | glibc_ver = commands.getoutput('ldd --version').splitlines()[0] |
| 697 | glibc_ver = re.search(r'(\d+\.\d+(\.\d+)?)', glibc_ver).group() |
| 698 | if utils.compare_versions(glibc_ver, ver) == -1: |
| 699 | raise error.TestError("Glibc too old (%s). Glibc >= %s is needed." % |
| 700 | (glibc_ver, ver)) |
| 701 | |
| 702 | def check_kernel_ver(ver): |
| 703 | kernel_ver = utils.system_output('uname -r') |
| 704 | kv_tmp = re.split(r'[-]', kernel_ver)[0:3] |
| 705 | # In compare_versions, if v1 < v2, return value == -1 |
| 706 | if utils.compare_versions(kv_tmp[0], ver) == -1: |
| 707 | raise error.TestError("Kernel too old (%s). Kernel > %s is needed." % |
| 708 | (kernel_ver, ver)) |
| 709 | |
| 710 | |
| 711 | def human_format(number): |
| 712 | # Convert number to kilo / mega / giga format. |
| 713 | if number < 1024: |
| 714 | return "%d" % number |
| 715 | kilo = float(number) / 1024.0 |
| 716 | if kilo < 1024: |
| 717 | return "%.2fk" % kilo |
| 718 | meg = kilo / 1024.0 |
| 719 | if meg < 1024: |
| 720 | return "%.2fM" % meg |
| 721 | gig = meg / 1024.0 |
| 722 | return "%.2fG" % gig |
| 723 | |
| 724 | |
| 725 | def numa_nodes(): |
| 726 | node_paths = glob.glob('/sys/devices/system/node/node*') |
| 727 | nodes = [int(re.sub(r'.*node(\d+)', r'\1', x)) for x in node_paths] |
| 728 | return (sorted(nodes)) |
| 729 | |
| 730 | |
| 731 | def node_size(): |
| 732 | nodes = max(len(numa_nodes()), 1) |
| 733 | return ((memtotal() * 1024) / nodes) |
| 734 | |
| 735 | |
| 736 | def pickle_load(filename): |
| 737 | return pickle.load(open(filename, 'r')) |
| 738 | |
| 739 | |
| 740 | # Return the kernel version and build timestamp. |
| 741 | def running_os_release(): |
| 742 | return os.uname()[2:4] |
| 743 | |
| 744 | |
| 745 | def running_os_ident(): |
| 746 | (version, timestamp) = running_os_release() |
| 747 | return version + '::' + timestamp |
| 748 | |
| 749 | |
| 750 | def running_os_full_version(): |
| 751 | (version, timestamp) = running_os_release() |
| 752 | return version |
| 753 | |
| 754 | |
| 755 | # much like find . -name 'pattern' |
| 756 | def locate(pattern, root=os.getcwd()): |
| 757 | for path, dirs, files in os.walk(root): |
| 758 | for f in files: |
| 759 | if fnmatch.fnmatch(f, pattern): |
| 760 | yield os.path.abspath(os.path.join(path, f)) |
| 761 | |
| 762 | |
| 763 | def freespace(path): |
| 764 | """Return the disk free space, in bytes""" |
| 765 | s = os.statvfs(path) |
| 766 | return s.f_bavail * s.f_bsize |
| 767 | |
| 768 | |
| 769 | def disk_block_size(path): |
| 770 | """Return the disk block size, in bytes""" |
| 771 | return os.statvfs(path).f_bsize |
| 772 | |
| 773 | |
| 774 | _DISK_PARTITION_3_RE = re.compile(r'^(/dev/hd[a-z]+)3', re.M) |
| 775 | |
| 776 | def get_disks(): |
| 777 | df_output = utils.system_output('df') |
| 778 | return _DISK_PARTITION_3_RE.findall(df_output) |
| 779 | |
| 780 | |
| 781 | def get_disk_size(disk_name): |
| 782 | """ |
| 783 | Return size of disk in byte. Return 0 in Error Case |
| 784 | |
| 785 | @param disk_name: disk name to find size |
| 786 | """ |
| 787 | device = os.path.basename(disk_name) |
| 788 | for line in file('/proc/partitions'): |
| 789 | try: |
| 790 | _, _, blocks, name = re.split(r' +', line.strip()) |
| 791 | except ValueError: |
| 792 | continue |
| 793 | if name == device: |
| 794 | return 1024 * int(blocks) |
| 795 | return 0 |
| 796 | |
| 797 | |
| 798 | def get_disk_size_gb(disk_name): |
| 799 | """ |
| 800 | Return size of disk in GB (10^9). Return 0 in Error Case |
| 801 | |
| 802 | @param disk_name: disk name to find size |
| 803 | """ |
| 804 | return int(get_disk_size(disk_name) / (10.0 ** 9) + 0.5) |
| 805 | |
| 806 | |
| 807 | def get_disk_model(disk_name): |
| 808 | """ |
| 809 | Return model name for internal storage device |
| 810 | |
| 811 | @param disk_name: disk name to find model |
| 812 | """ |
| 813 | cmd1 = 'udevadm info --query=property --name=%s' % disk_name |
| 814 | cmd2 = 'grep -E "ID_(NAME|MODEL)="' |
| 815 | cmd3 = 'cut -f 2 -d"="' |
| 816 | cmd = ' | '.join([cmd1, cmd2, cmd3]) |
| 817 | return utils.system_output(cmd) |
| 818 | |
| 819 | |
Gwendal Grignou | 876e691 | 2017-06-21 12:17:36 -0700 | [diff] [blame] | 820 | _DISK_DEV_RE = re.compile(r'/dev/sd[a-z]|' |
| 821 | r'/dev/mmcblk[0-9]+|' |
| 822 | r'/dev/nvme[0-9]+n[0-9]+') |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 823 | |
| 824 | |
| 825 | def get_disk_from_filename(filename): |
| 826 | """ |
| 827 | Return the disk device the filename is on. |
| 828 | If the file is on tmpfs or other special file systems, |
| 829 | return None. |
| 830 | |
| 831 | @param filename: name of file, full path. |
| 832 | """ |
| 833 | |
| 834 | if not os.path.exists(filename): |
| 835 | raise error.TestError('file %s missing' % filename) |
| 836 | |
| 837 | if filename[0] != '/': |
| 838 | raise error.TestError('This code works only with full path') |
| 839 | |
| 840 | m = _DISK_DEV_RE.match(filename) |
| 841 | while not m: |
| 842 | if filename[0] != '/': |
| 843 | return None |
| 844 | if filename == '/dev/root': |
| 845 | cmd = 'rootdev -d -s' |
| 846 | elif filename.startswith('/dev/mapper'): |
| 847 | cmd = 'dmsetup table "%s"' % os.path.basename(filename) |
| 848 | dmsetup_output = utils.system_output(cmd).split(' ') |
| 849 | if dmsetup_output[2] == 'verity': |
| 850 | maj_min = dmsetup_output[4] |
| 851 | elif dmsetup_output[2] == 'crypt': |
| 852 | maj_min = dmsetup_output[6] |
| 853 | cmd = 'realpath "/dev/block/%s"' % maj_min |
| 854 | elif filename.startswith('/dev/loop'): |
| 855 | cmd = 'losetup -O BACK-FILE "%s" | tail -1' % filename |
| 856 | else: |
| 857 | cmd = 'df "%s" | tail -1 | cut -f 1 -d" "' % filename |
| 858 | filename = utils.system_output(cmd) |
| 859 | m = _DISK_DEV_RE.match(filename) |
| 860 | return m.group(0) |
| 861 | |
| 862 | |
| 863 | def get_disk_firmware_version(disk_name): |
| 864 | """ |
| 865 | Return firmware version for internal storage device. (empty string for eMMC) |
| 866 | |
| 867 | @param disk_name: disk name to find model |
| 868 | """ |
| 869 | cmd1 = 'udevadm info --query=property --name=%s' % disk_name |
| 870 | cmd2 = 'grep -E "ID_REVISION="' |
| 871 | cmd3 = 'cut -f 2 -d"="' |
| 872 | cmd = ' | '.join([cmd1, cmd2, cmd3]) |
| 873 | return utils.system_output(cmd) |
| 874 | |
| 875 | |
| 876 | def is_disk_scsi(disk_name): |
| 877 | """ |
| 878 | Return true if disk is a scsi device, return false otherwise |
| 879 | |
| 880 | @param disk_name: disk name check |
| 881 | """ |
| 882 | return re.match('/dev/sd[a-z]+', disk_name) |
| 883 | |
| 884 | |
| 885 | def is_disk_harddisk(disk_name): |
| 886 | """ |
| 887 | Return true if disk is a harddisk, return false otherwise |
| 888 | |
| 889 | @param disk_name: disk name check |
| 890 | """ |
| 891 | cmd1 = 'udevadm info --query=property --name=%s' % disk_name |
| 892 | cmd2 = 'grep -E "ID_ATA_ROTATION_RATE_RPM="' |
| 893 | cmd3 = 'cut -f 2 -d"="' |
| 894 | cmd = ' | '.join([cmd1, cmd2, cmd3]) |
| 895 | |
| 896 | rtt = utils.system_output(cmd) |
| 897 | |
| 898 | # eMMC will not have this field; rtt == '' |
| 899 | # SSD will have zero rotation rate; rtt == '0' |
| 900 | # For harddisk rtt > 0 |
| 901 | return rtt and int(rtt) > 0 |
| 902 | |
Gwendal Grignou | 427c7b2 | 2017-10-26 17:39:29 -0700 | [diff] [blame] | 903 | def concat_partition(disk_name, partition_number): |
| 904 | """ |
| 905 | Return the name of a partition: |
| 906 | sda, 3 --> sda3 |
| 907 | mmcblk0, 3 --> mmcblk0p3 |
| 908 | |
| 909 | @param disk_name: diskname string |
| 910 | @param partition_number: integer |
| 911 | """ |
| 912 | if disk_name.endswith(tuple(str(i) for i in range(0, 10))): |
| 913 | sep = 'p' |
| 914 | else: |
| 915 | sep = '' |
| 916 | return disk_name + sep + str(partition_number) |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 917 | |
| 918 | def verify_hdparm_feature(disk_name, feature): |
| 919 | """ |
| 920 | Check for feature support for SCSI disk using hdparm |
| 921 | |
| 922 | @param disk_name: target disk |
| 923 | @param feature: hdparm output string of the feature |
| 924 | """ |
| 925 | cmd = 'hdparm -I %s | grep -q "%s"' % (disk_name, feature) |
| 926 | ret = utils.system(cmd, ignore_status=True) |
| 927 | if ret == 0: |
| 928 | return True |
| 929 | elif ret == 1: |
| 930 | return False |
| 931 | else: |
| 932 | raise error.TestFail('Error running command %s' % cmd) |
| 933 | |
| 934 | |
| 935 | def get_storage_error_msg(disk_name, reason): |
| 936 | """ |
| 937 | Get Error message for storage test which include disk model. |
| 938 | and also include the firmware version for the SCSI disk |
| 939 | |
| 940 | @param disk_name: target disk |
| 941 | @param reason: Reason of the error. |
| 942 | """ |
| 943 | |
| 944 | msg = reason |
| 945 | |
| 946 | model = get_disk_model(disk_name) |
| 947 | msg += ' Disk model: %s' % model |
| 948 | |
| 949 | if is_disk_scsi(disk_name): |
| 950 | fw = get_disk_firmware_version(disk_name) |
| 951 | msg += ' firmware: %s' % fw |
| 952 | |
| 953 | return msg |
| 954 | |
| 955 | |
| 956 | def load_module(module_name, params=None): |
| 957 | # Checks if a module has already been loaded |
| 958 | if module_is_loaded(module_name): |
| 959 | return False |
| 960 | |
| 961 | cmd = '/sbin/modprobe ' + module_name |
| 962 | if params: |
| 963 | cmd += ' ' + params |
| 964 | utils.system(cmd) |
| 965 | return True |
| 966 | |
| 967 | |
| 968 | def unload_module(module_name): |
| 969 | """ |
| 970 | Removes a module. Handles dependencies. If even then it's not possible |
| 971 | to remove one of the modules, it will trhow an error.CmdError exception. |
| 972 | |
| 973 | @param module_name: Name of the module we want to remove. |
| 974 | """ |
| 975 | l_raw = utils.system_output("/bin/lsmod").splitlines() |
| 976 | lsmod = [x for x in l_raw if x.split()[0] == module_name] |
| 977 | if len(lsmod) > 0: |
| 978 | line_parts = lsmod[0].split() |
| 979 | if len(line_parts) == 4: |
| 980 | submodules = line_parts[3].split(",") |
| 981 | for submodule in submodules: |
| 982 | unload_module(submodule) |
| 983 | utils.system("/sbin/modprobe -r %s" % module_name) |
| 984 | logging.info("Module %s unloaded", module_name) |
| 985 | else: |
| 986 | logging.info("Module %s is already unloaded", module_name) |
| 987 | |
| 988 | |
| 989 | def module_is_loaded(module_name): |
| 990 | module_name = module_name.replace('-', '_') |
| 991 | modules = utils.system_output('/bin/lsmod').splitlines() |
| 992 | for module in modules: |
| 993 | if module.startswith(module_name) and module[len(module_name)] == ' ': |
| 994 | return True |
| 995 | return False |
| 996 | |
| 997 | |
| 998 | def get_loaded_modules(): |
| 999 | lsmod_output = utils.system_output('/bin/lsmod').splitlines()[1:] |
| 1000 | return [line.split(None, 1)[0] for line in lsmod_output] |
| 1001 | |
| 1002 | |
| 1003 | def get_huge_page_size(): |
| 1004 | output = utils.system_output('grep Hugepagesize /proc/meminfo') |
| 1005 | return int(output.split()[1]) # Assumes units always in kB. :( |
| 1006 | |
| 1007 | |
| 1008 | def get_num_huge_pages(): |
| 1009 | raw_hugepages = utils.system_output('/sbin/sysctl vm.nr_hugepages') |
| 1010 | return int(raw_hugepages.split()[2]) |
| 1011 | |
| 1012 | |
| 1013 | def set_num_huge_pages(num): |
| 1014 | utils.system('/sbin/sysctl vm.nr_hugepages=%d' % num) |
| 1015 | |
| 1016 | |
| 1017 | def ping_default_gateway(): |
| 1018 | """Ping the default gateway.""" |
| 1019 | |
| 1020 | network = open('/etc/sysconfig/network') |
| 1021 | m = re.search('GATEWAY=(\S+)', network.read()) |
| 1022 | |
| 1023 | if m: |
| 1024 | gw = m.group(1) |
| 1025 | cmd = 'ping %s -c 5 > /dev/null' % gw |
| 1026 | return utils.system(cmd, ignore_status=True) |
| 1027 | |
| 1028 | raise error.TestError('Unable to find default gateway') |
| 1029 | |
| 1030 | |
| 1031 | def drop_caches(): |
| 1032 | """Writes back all dirty pages to disk and clears all the caches.""" |
| 1033 | utils.system("sync") |
| 1034 | # We ignore failures here as this will fail on 2.6.11 kernels. |
| 1035 | utils.system("echo 3 > /proc/sys/vm/drop_caches", ignore_status=True) |
| 1036 | |
| 1037 | |
| 1038 | def process_is_alive(name_pattern): |
| 1039 | """ |
| 1040 | 'pgrep name' misses all python processes and also long process names. |
| 1041 | 'pgrep -f name' gets all shell commands with name in args. |
| 1042 | So look only for command whose initial pathname ends with name. |
| 1043 | Name itself is an egrep pattern, so it can use | etc for variations. |
| 1044 | """ |
| 1045 | return utils.system("pgrep -f '^([^ /]*/)*(%s)([ ]|$)'" % name_pattern, |
| 1046 | ignore_status=True) == 0 |
| 1047 | |
| 1048 | |
| 1049 | def get_hwclock_seconds(utc=True): |
| 1050 | """ |
| 1051 | Return the hardware clock in seconds as a floating point value. |
| 1052 | Use Coordinated Universal Time if utc is True, local time otherwise. |
| 1053 | Raise a ValueError if unable to read the hardware clock. |
| 1054 | """ |
| 1055 | cmd = '/sbin/hwclock --debug' |
| 1056 | if utc: |
| 1057 | cmd += ' --utc' |
| 1058 | hwclock_output = utils.system_output(cmd, ignore_status=True) |
| 1059 | match = re.search(r'= ([0-9]+) seconds since .+ (-?[0-9.]+) seconds$', |
| 1060 | hwclock_output, re.DOTALL) |
| 1061 | if match: |
| 1062 | seconds = int(match.group(1)) + float(match.group(2)) |
| 1063 | logging.debug('hwclock seconds = %f', seconds) |
| 1064 | return seconds |
| 1065 | |
| 1066 | raise ValueError('Unable to read the hardware clock -- ' + |
| 1067 | hwclock_output) |
| 1068 | |
| 1069 | |
| 1070 | def set_wake_alarm(alarm_time): |
| 1071 | """ |
| 1072 | Set the hardware RTC-based wake alarm to 'alarm_time'. |
| 1073 | """ |
| 1074 | utils.write_one_line('/sys/class/rtc/rtc0/wakealarm', str(alarm_time)) |
| 1075 | |
| 1076 | |
| 1077 | def set_power_state(state): |
| 1078 | """ |
| 1079 | Set the system power state to 'state'. |
| 1080 | """ |
| 1081 | utils.write_one_line('/sys/power/state', state) |
| 1082 | |
| 1083 | |
| 1084 | def standby(): |
| 1085 | """ |
| 1086 | Power-on suspend (S1) |
| 1087 | """ |
| 1088 | set_power_state('standby') |
| 1089 | |
| 1090 | |
| 1091 | def suspend_to_ram(): |
| 1092 | """ |
| 1093 | Suspend the system to RAM (S3) |
| 1094 | """ |
| 1095 | set_power_state('mem') |
| 1096 | |
| 1097 | |
| 1098 | def suspend_to_disk(): |
| 1099 | """ |
| 1100 | Suspend the system to disk (S4) |
| 1101 | """ |
| 1102 | set_power_state('disk') |
| 1103 | |
| 1104 | |
Po-Hsien Wang | c02992f | 2017-08-10 11:18:46 -0700 | [diff] [blame] | 1105 | _AUTOTEST_CLIENT_PATH = os.path.join(os.path.dirname(__file__), '..') |
| 1106 | _AMD_PCI_IDS_FILE_PATH = os.path.join(_AUTOTEST_CLIENT_PATH, |
| 1107 | 'bin/amd_pci_ids.json') |
| 1108 | _INTEL_PCI_IDS_FILE_PATH = os.path.join(_AUTOTEST_CLIENT_PATH, |
| 1109 | 'bin/intel_pci_ids.json') |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1110 | _UI_USE_FLAGS_FILE_PATH = '/etc/ui_use_flags.txt' |
| 1111 | |
| 1112 | # Command to check if a package is installed. If the package is not installed |
| 1113 | # the command shall fail. |
| 1114 | _CHECK_PACKAGE_INSTALLED_COMMAND =( |
| 1115 | "dpkg-query -W -f='${Status}\n' %s | head -n1 | awk '{print $3;}' | " |
| 1116 | "grep -q '^installed$'") |
| 1117 | |
| 1118 | pciid_to_amd_architecture = {} |
| 1119 | pciid_to_intel_architecture = {} |
| 1120 | |
| 1121 | class Crossystem(object): |
| 1122 | """A wrapper for the crossystem utility.""" |
| 1123 | |
| 1124 | def __init__(self, client): |
| 1125 | self.cros_system_data = {} |
| 1126 | self._client = client |
| 1127 | |
| 1128 | def init(self): |
| 1129 | self.cros_system_data = {} |
| 1130 | (_, fname) = tempfile.mkstemp() |
| 1131 | f = open(fname, 'w') |
| 1132 | self._client.run('crossystem', stdout_tee=f) |
| 1133 | f.close() |
| 1134 | text = utils.read_file(fname) |
| 1135 | for line in text.splitlines(): |
| 1136 | assignment_string = line.split('#')[0] |
| 1137 | if not assignment_string.count('='): |
| 1138 | continue |
| 1139 | (name, value) = assignment_string.split('=', 1) |
| 1140 | self.cros_system_data[name.strip()] = value.strip() |
| 1141 | os.remove(fname) |
| 1142 | |
| 1143 | def __getattr__(self, name): |
| 1144 | """ |
| 1145 | Retrieve a crosssystem attribute. |
| 1146 | |
| 1147 | The call crossystemobject.name() will return the crossystem reported |
| 1148 | string. |
| 1149 | """ |
| 1150 | return lambda: self.cros_system_data[name] |
| 1151 | |
| 1152 | |
| 1153 | def get_oldest_pid_by_name(name): |
| 1154 | """ |
| 1155 | Return the oldest pid of a process whose name perfectly matches |name|. |
| 1156 | |
| 1157 | name is an egrep expression, which will be matched against the entire name |
| 1158 | of processes on the system. For example: |
| 1159 | |
| 1160 | get_oldest_pid_by_name('chrome') |
| 1161 | |
| 1162 | on a system running |
| 1163 | 8600 ? 00:00:04 chrome |
| 1164 | 8601 ? 00:00:00 chrome |
| 1165 | 8602 ? 00:00:00 chrome-sandbox |
| 1166 | |
| 1167 | would return 8600, as that's the oldest process that matches. |
| 1168 | chrome-sandbox would not be matched. |
| 1169 | |
| 1170 | Arguments: |
| 1171 | name: egrep expression to match. Will be anchored at the beginning and |
| 1172 | end of the match string. |
| 1173 | |
| 1174 | Returns: |
| 1175 | pid as an integer, or None if one cannot be found. |
| 1176 | |
| 1177 | Raises: |
| 1178 | ValueError if pgrep returns something odd. |
| 1179 | """ |
| 1180 | str_pid = utils.system_output('pgrep -o ^%s$' % name, |
| 1181 | ignore_status=True).rstrip() |
| 1182 | if str_pid: |
| 1183 | return int(str_pid) |
| 1184 | |
| 1185 | |
| 1186 | def get_oldest_by_name(name): |
| 1187 | """Return pid and command line of oldest process whose name matches |name|. |
| 1188 | |
| 1189 | @param name: egrep expression to match desired process name. |
| 1190 | @return: A tuple of (pid, command_line) of the oldest process whose name |
| 1191 | matches |name|. |
| 1192 | |
| 1193 | """ |
| 1194 | pid = get_oldest_pid_by_name(name) |
| 1195 | if pid: |
| 1196 | command_line = utils.system_output('ps -p %i -o command=' % pid, |
| 1197 | ignore_status=True).rstrip() |
| 1198 | return (pid, command_line) |
| 1199 | |
| 1200 | |
| 1201 | def get_chrome_remote_debugging_port(): |
| 1202 | """Returns remote debugging port for Chrome. |
| 1203 | |
| 1204 | Parse chrome process's command line argument to get the remote debugging |
Achuith Bhandarkar | e3adb0a | 2017-08-10 14:15:57 -0700 | [diff] [blame] | 1205 | port. if it is 0, look at DevToolsActivePort for the ephemeral port. |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1206 | """ |
| 1207 | _, command = get_oldest_by_name('chrome') |
| 1208 | matches = re.search('--remote-debugging-port=([0-9]+)', command) |
Achuith Bhandarkar | e3adb0a | 2017-08-10 14:15:57 -0700 | [diff] [blame] | 1209 | if not matches: |
| 1210 | return 0 |
| 1211 | port = int(matches.group(1)) |
| 1212 | if port: |
| 1213 | return port |
| 1214 | with open('/home/chronos/DevToolsActivePort') as f: |
| 1215 | return int(f.readline().rstrip()) |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1216 | |
| 1217 | |
| 1218 | def get_process_list(name, command_line=None): |
| 1219 | """ |
| 1220 | Return the list of pid for matching process |name command_line|. |
| 1221 | |
| 1222 | on a system running |
| 1223 | 31475 ? 0:06 /opt/google/chrome/chrome --allow-webui-compositing - |
| 1224 | 31478 ? 0:00 /opt/google/chrome/chrome-sandbox /opt/google/chrome/ |
| 1225 | 31485 ? 0:00 /opt/google/chrome/chrome --type=zygote --log-level=1 |
| 1226 | 31532 ? 1:05 /opt/google/chrome/chrome --type=renderer |
| 1227 | |
| 1228 | get_process_list('chrome') |
| 1229 | would return ['31475', '31485', '31532'] |
| 1230 | |
| 1231 | get_process_list('chrome', '--type=renderer') |
| 1232 | would return ['31532'] |
| 1233 | |
| 1234 | Arguments: |
| 1235 | name: process name to search for. If command_line is provided, name is |
| 1236 | matched against full command line. If command_line is not provided, |
| 1237 | name is only matched against the process name. |
| 1238 | command line: when command line is passed, the full process command line |
| 1239 | is used for matching. |
| 1240 | |
| 1241 | Returns: |
| 1242 | list of PIDs of the matching processes. |
| 1243 | |
| 1244 | """ |
| 1245 | # TODO(rohitbm) crbug.com/268861 |
| 1246 | flag = '-x' if not command_line else '-f' |
| 1247 | name = '\'%s.*%s\'' % (name, command_line) if command_line else name |
| 1248 | str_pid = utils.system_output('pgrep %s %s' % (flag, name), |
| 1249 | ignore_status=True).rstrip() |
| 1250 | return str_pid.split() |
| 1251 | |
| 1252 | |
| 1253 | def nuke_process_by_name(name, with_prejudice=False): |
| 1254 | """Tell the oldest process specified by name to exit. |
| 1255 | |
| 1256 | Arguments: |
| 1257 | name: process name specifier, as understood by pgrep. |
| 1258 | with_prejudice: if True, don't allow for graceful exit. |
| 1259 | |
| 1260 | Raises: |
| 1261 | error.AutoservPidAlreadyDeadError: no existing process matches name. |
| 1262 | """ |
| 1263 | try: |
| 1264 | pid = get_oldest_pid_by_name(name) |
| 1265 | except Exception as e: |
| 1266 | logging.error(e) |
| 1267 | return |
| 1268 | if pid is None: |
| 1269 | raise error.AutoservPidAlreadyDeadError('No process matching %s.' % |
| 1270 | name) |
| 1271 | if with_prejudice: |
| 1272 | utils.nuke_pid(pid, [signal.SIGKILL]) |
| 1273 | else: |
| 1274 | utils.nuke_pid(pid) |
| 1275 | |
| 1276 | |
| 1277 | def ensure_processes_are_dead_by_name(name, timeout_sec=10): |
| 1278 | """Terminate all processes specified by name and ensure they're gone. |
| 1279 | |
| 1280 | Arguments: |
| 1281 | name: process name specifier, as understood by pgrep. |
| 1282 | timeout_sec: maximum number of seconds to wait for processes to die. |
| 1283 | |
| 1284 | Raises: |
| 1285 | error.AutoservPidAlreadyDeadError: no existing process matches name. |
Allen Li | 5ed7e63 | 2017-02-03 16:31:33 -0800 | [diff] [blame] | 1286 | utils.TimeoutError: if processes still exist after timeout_sec. |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1287 | """ |
| 1288 | |
| 1289 | def list_and_kill_processes(name): |
| 1290 | process_list = get_process_list(name) |
| 1291 | try: |
| 1292 | for pid in [int(str_pid) for str_pid in process_list]: |
| 1293 | utils.nuke_pid(pid) |
| 1294 | except error.AutoservPidAlreadyDeadError: |
| 1295 | pass |
| 1296 | return process_list |
| 1297 | |
| 1298 | utils.poll_for_condition(lambda: list_and_kill_processes(name) == [], |
| 1299 | timeout=timeout_sec) |
| 1300 | |
| 1301 | |
| 1302 | def is_virtual_machine(): |
| 1303 | return 'QEMU' in platform.processor() |
| 1304 | |
| 1305 | |
| 1306 | def save_vm_state(checkpoint): |
| 1307 | """Saves the current state of the virtual machine. |
| 1308 | |
| 1309 | This function is a NOOP if the test is not running under a virtual machine |
| 1310 | with the USB serial port redirected. |
| 1311 | |
| 1312 | Arguments: |
| 1313 | checkpoint - Name used to identify this state |
| 1314 | |
| 1315 | Returns: |
| 1316 | None |
| 1317 | """ |
| 1318 | # The QEMU monitor has been redirected to the guest serial port located at |
| 1319 | # /dev/ttyUSB0. To save the state of the VM, we just send the 'savevm' |
| 1320 | # command to the serial port. |
| 1321 | if is_virtual_machine() and os.path.exists('/dev/ttyUSB0'): |
| 1322 | logging.info('Saving VM state "%s"', checkpoint) |
| 1323 | serial = open('/dev/ttyUSB0', 'w') |
| 1324 | serial.write('savevm %s\r\n' % checkpoint) |
| 1325 | logging.info('Done saving VM state "%s"', checkpoint) |
| 1326 | |
| 1327 | |
| 1328 | def check_raw_dmesg(dmesg, message_level, whitelist): |
| 1329 | """Checks dmesg for unexpected warnings. |
| 1330 | |
| 1331 | This function parses dmesg for message with message_level <= message_level |
| 1332 | which do not appear in the whitelist. |
| 1333 | |
| 1334 | Arguments: |
| 1335 | dmesg - string containing raw dmesg buffer |
| 1336 | message_level - minimum message priority to check |
| 1337 | whitelist - messages to ignore |
| 1338 | |
| 1339 | Returns: |
| 1340 | List of unexpected warnings |
| 1341 | """ |
| 1342 | whitelist_re = re.compile(r'(%s)' % '|'.join(whitelist)) |
| 1343 | unexpected = [] |
| 1344 | for line in dmesg.splitlines(): |
| 1345 | if int(line[1]) <= message_level: |
| 1346 | stripped_line = line.split('] ', 1)[1] |
| 1347 | if whitelist_re.search(stripped_line): |
| 1348 | continue |
| 1349 | unexpected.append(stripped_line) |
| 1350 | return unexpected |
| 1351 | |
| 1352 | |
| 1353 | def verify_mesg_set(mesg, regex, whitelist): |
| 1354 | """Verifies that the exact set of messages are present in a text. |
| 1355 | |
| 1356 | This function finds all strings in the text matching a certain regex, and |
| 1357 | then verifies that all expected strings are present in the set, and no |
| 1358 | unexpected strings are there. |
| 1359 | |
| 1360 | Arguments: |
| 1361 | mesg - the mutiline text to be scanned |
| 1362 | regex - regular expression to match |
| 1363 | whitelist - messages to find in the output, a list of strings |
| 1364 | (potentially regexes) to look for in the filtered output. All these |
| 1365 | strings must be there, and no other strings should be present in the |
| 1366 | filtered output. |
| 1367 | |
| 1368 | Returns: |
| 1369 | string of inconsistent findings (i.e. an empty string on success). |
| 1370 | """ |
| 1371 | |
| 1372 | rv = [] |
| 1373 | |
| 1374 | missing_strings = [] |
| 1375 | present_strings = [] |
| 1376 | for line in mesg.splitlines(): |
| 1377 | if not re.search(r'%s' % regex, line): |
| 1378 | continue |
| 1379 | present_strings.append(line.split('] ', 1)[1]) |
| 1380 | |
| 1381 | for string in whitelist: |
| 1382 | for present_string in list(present_strings): |
| 1383 | if re.search(r'^%s$' % string, present_string): |
| 1384 | present_strings.remove(present_string) |
| 1385 | break |
| 1386 | else: |
| 1387 | missing_strings.append(string) |
| 1388 | |
| 1389 | if present_strings: |
| 1390 | rv.append('unexpected strings:') |
| 1391 | rv.extend(present_strings) |
| 1392 | if missing_strings: |
| 1393 | rv.append('missing strings:') |
| 1394 | rv.extend(missing_strings) |
| 1395 | |
| 1396 | return '\n'.join(rv) |
| 1397 | |
| 1398 | |
| 1399 | def target_is_pie(): |
| 1400 | """Returns whether the toolchain produces a PIE (position independent |
| 1401 | executable) by default. |
| 1402 | |
| 1403 | Arguments: |
| 1404 | None |
| 1405 | |
| 1406 | Returns: |
| 1407 | True if the target toolchain produces a PIE by default. |
| 1408 | False otherwise. |
| 1409 | """ |
| 1410 | |
| 1411 | command = 'echo | ${CC} -E -dD -P - | grep -i pie' |
| 1412 | result = utils.system_output(command, |
| 1413 | retain_output=True, |
| 1414 | ignore_status=True) |
| 1415 | if re.search('#define __PIE__', result): |
| 1416 | return True |
| 1417 | else: |
| 1418 | return False |
| 1419 | |
| 1420 | |
| 1421 | def target_is_x86(): |
| 1422 | """Returns whether the toolchain produces an x86 object |
| 1423 | |
| 1424 | Arguments: |
| 1425 | None |
| 1426 | |
| 1427 | Returns: |
| 1428 | True if the target toolchain produces an x86 object |
| 1429 | False otherwise. |
| 1430 | """ |
| 1431 | |
| 1432 | command = 'echo | ${CC} -E -dD -P - | grep -i 86' |
| 1433 | result = utils.system_output(command, |
| 1434 | retain_output=True, |
| 1435 | ignore_status=True) |
| 1436 | if re.search('__i386__', result) or re.search('__x86_64__', result): |
| 1437 | return True |
| 1438 | else: |
| 1439 | return False |
| 1440 | |
| 1441 | |
| 1442 | def mounts(): |
| 1443 | ret = [] |
| 1444 | for line in file('/proc/mounts'): |
| 1445 | m = re.match( |
| 1446 | r'(?P<src>\S+) (?P<dest>\S+) (?P<type>\S+) (?P<opts>\S+).*', line) |
| 1447 | if m: |
| 1448 | ret.append(m.groupdict()) |
| 1449 | return ret |
| 1450 | |
| 1451 | |
| 1452 | def is_mountpoint(path): |
| 1453 | return path in [m['dest'] for m in mounts()] |
| 1454 | |
| 1455 | |
| 1456 | def require_mountpoint(path): |
| 1457 | """ |
| 1458 | Raises an exception if path is not a mountpoint. |
| 1459 | """ |
| 1460 | if not is_mountpoint(path): |
| 1461 | raise error.TestFail('Path not mounted: "%s"' % path) |
| 1462 | |
| 1463 | |
| 1464 | def random_username(): |
| 1465 | return str(uuid.uuid4()) + '@example.com' |
| 1466 | |
| 1467 | |
| 1468 | def get_signin_credentials(filepath): |
| 1469 | """Returns user_id, password tuple from credentials file at filepath. |
| 1470 | |
| 1471 | File must have one line of the format user_id:password |
| 1472 | |
| 1473 | @param filepath: path of credentials file. |
| 1474 | @return user_id, password tuple. |
| 1475 | """ |
| 1476 | user_id, password = None, None |
| 1477 | if os.path.isfile(filepath): |
| 1478 | with open(filepath) as f: |
| 1479 | user_id, password = f.read().rstrip().split(':') |
| 1480 | return user_id, password |
| 1481 | |
| 1482 | |
| 1483 | def parse_cmd_output(command, run_method=utils.run): |
| 1484 | """Runs a command on a host object to retrieve host attributes. |
| 1485 | |
| 1486 | The command should output to stdout in the format of: |
| 1487 | <key> = <value> # <optional_comment> |
| 1488 | |
| 1489 | |
| 1490 | @param command: Command to execute on the host. |
| 1491 | @param run_method: Function to use to execute the command. Defaults to |
| 1492 | utils.run so that the command will be executed locally. |
| 1493 | Can be replace with a host.run call so that it will |
| 1494 | execute on a DUT or external machine. Method must accept |
| 1495 | a command argument, stdout_tee and stderr_tee args and |
| 1496 | return a result object with a string attribute stdout |
| 1497 | which will be parsed. |
| 1498 | |
| 1499 | @returns a dictionary mapping host attributes to their values. |
| 1500 | """ |
| 1501 | result = {} |
| 1502 | # Suppresses stdout so that the files are not printed to the logs. |
| 1503 | cmd_result = run_method(command, stdout_tee=None, stderr_tee=None) |
| 1504 | for line in cmd_result.stdout.splitlines(): |
| 1505 | # Lines are of the format "<key> = <value> # <comment>" |
| 1506 | key_value = re.match(r'^\s*(?P<key>[^ ]+)\s*=\s*(?P<value>[^ ' |
| 1507 | r']+)(?:\s*#.*)?$', line) |
| 1508 | if key_value: |
| 1509 | result[key_value.group('key')] = key_value.group('value') |
| 1510 | return result |
| 1511 | |
| 1512 | |
| 1513 | def set_from_keyval_output(out, delimiter=' '): |
| 1514 | """Parse delimiter-separated key-val output into a set of tuples. |
| 1515 | |
| 1516 | Output is expected to be multiline text output from a command. |
| 1517 | Stuffs the key-vals into tuples in a set to be later compared. |
| 1518 | |
| 1519 | e.g. deactivated 0 |
| 1520 | disableForceClear 0 |
| 1521 | ==> set(('deactivated', '0'), ('disableForceClear', '0')) |
| 1522 | |
| 1523 | @param out: multiple lines of space-separated key-val pairs. |
| 1524 | @param delimiter: character that separates key from val. Usually a |
| 1525 | space but may be '=' or something else. |
| 1526 | @return set of key-val tuples. |
| 1527 | """ |
| 1528 | results = set() |
| 1529 | kv_match_re = re.compile('([^ ]+)%s(.*)' % delimiter) |
| 1530 | for linecr in out.splitlines(): |
| 1531 | match = kv_match_re.match(linecr.strip()) |
| 1532 | if match: |
| 1533 | results.add((match.group(1), match.group(2))) |
| 1534 | return results |
| 1535 | |
| 1536 | |
| 1537 | def get_cpu_usage(): |
| 1538 | """Returns machine's CPU usage. |
| 1539 | |
| 1540 | This function uses /proc/stat to identify CPU usage. |
| 1541 | Returns: |
Kristoffer Erlandsson | 82b9f32 | 2017-11-07 12:20:38 +0100 | [diff] [blame] | 1542 | A dictionary with values for all columns in /proc/stat |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1543 | Sample dictionary: |
| 1544 | { |
| 1545 | 'user': 254544, |
| 1546 | 'nice': 9, |
| 1547 | 'system': 254768, |
| 1548 | 'idle': 2859878, |
Kristoffer Erlandsson | 82b9f32 | 2017-11-07 12:20:38 +0100 | [diff] [blame] | 1549 | 'iowait': 1, |
| 1550 | 'irq': 2, |
| 1551 | 'softirq': 3, |
| 1552 | 'steal': 4, |
| 1553 | 'guest': 5, |
| 1554 | 'guest_nice': 6 |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1555 | } |
Kristoffer Erlandsson | 82b9f32 | 2017-11-07 12:20:38 +0100 | [diff] [blame] | 1556 | If a column is missing or malformed in /proc/stat (typically on older |
| 1557 | systems), the value for that column is set to 0. |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1558 | """ |
Kristoffer Erlandsson | 82b9f32 | 2017-11-07 12:20:38 +0100 | [diff] [blame] | 1559 | with _open_file('/proc/stat') as proc_stat: |
| 1560 | cpu_usage_str = proc_stat.readline().split() |
| 1561 | columns = ('user', 'nice', 'system', 'idle', 'iowait', 'irq', 'softirq', |
| 1562 | 'steal', 'guest', 'guest_nice') |
| 1563 | d = {} |
| 1564 | for index, col in enumerate(columns, 1): |
| 1565 | try: |
| 1566 | d[col] = int(cpu_usage_str[index]) |
| 1567 | except: |
| 1568 | d[col] = 0 |
| 1569 | return d |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1570 | |
| 1571 | def compute_active_cpu_time(cpu_usage_start, cpu_usage_end): |
| 1572 | """Computes the fraction of CPU time spent non-idling. |
| 1573 | |
| 1574 | This function should be invoked using before/after values from calls to |
| 1575 | get_cpu_usage(). |
Kristoffer Erlandsson | 82b9f32 | 2017-11-07 12:20:38 +0100 | [diff] [blame] | 1576 | |
| 1577 | See https://stackoverflow.com/a/23376195 and |
| 1578 | https://unix.stackexchange.com/a/303224 for some more context how |
| 1579 | to calculate usage given two /proc/stat snapshots. |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1580 | """ |
Kristoffer Erlandsson | 82b9f32 | 2017-11-07 12:20:38 +0100 | [diff] [blame] | 1581 | idle_cols = ('idle', 'iowait') # All other cols are calculated as active. |
| 1582 | time_active_start = sum([x[1] for x in cpu_usage_start.iteritems() |
| 1583 | if x[0] not in idle_cols]) |
| 1584 | time_active_end = sum([x[1] for x in cpu_usage_end.iteritems() |
| 1585 | if x[0] not in idle_cols]) |
| 1586 | total_time_start = sum(cpu_usage_start.values()) |
| 1587 | total_time_end = sum(cpu_usage_end.values()) |
Ilja H. Friedel | b2b28b3 | 2017-12-19 16:34:25 -0800 | [diff] [blame] | 1588 | # Avoid bogus division which has been observed on Tegra. |
| 1589 | if total_time_end <= total_time_start: |
| 1590 | logging.warning('compute_active_cpu_time observed bogus data') |
| 1591 | # We pretend to be busy, this will force a longer wait for idle CPU. |
| 1592 | return 1.0 |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1593 | return ((float(time_active_end) - time_active_start) / |
| 1594 | (total_time_end - total_time_start)) |
| 1595 | |
| 1596 | |
| 1597 | def is_pgo_mode(): |
| 1598 | return 'USE_PGO' in os.environ |
| 1599 | |
| 1600 | |
| 1601 | def wait_for_idle_cpu(timeout, utilization): |
| 1602 | """Waits for the CPU to become idle (< utilization). |
| 1603 | |
| 1604 | Args: |
| 1605 | timeout: The longest time in seconds to wait before throwing an error. |
| 1606 | utilization: The CPU usage below which the system should be considered |
| 1607 | idle (between 0 and 1.0 independent of cores/hyperthreads). |
| 1608 | """ |
| 1609 | time_passed = 0.0 |
| 1610 | fraction_active_time = 1.0 |
| 1611 | sleep_time = 1 |
| 1612 | logging.info('Starting to wait up to %.1fs for idle CPU...', timeout) |
| 1613 | while fraction_active_time >= utilization: |
| 1614 | cpu_usage_start = get_cpu_usage() |
| 1615 | # Split timeout interval into not too many chunks to limit log spew. |
| 1616 | # Start at 1 second, increase exponentially |
| 1617 | time.sleep(sleep_time) |
| 1618 | time_passed += sleep_time |
| 1619 | sleep_time = min(16.0, 2.0 * sleep_time) |
| 1620 | cpu_usage_end = get_cpu_usage() |
Ilja H. Friedel | b2b28b3 | 2017-12-19 16:34:25 -0800 | [diff] [blame] | 1621 | fraction_active_time = compute_active_cpu_time(cpu_usage_start, |
| 1622 | cpu_usage_end) |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1623 | logging.info('After waiting %.1fs CPU utilization is %.3f.', |
| 1624 | time_passed, fraction_active_time) |
| 1625 | if time_passed > timeout: |
| 1626 | logging.warning('CPU did not become idle.') |
| 1627 | log_process_activity() |
| 1628 | # crosbug.com/37389 |
| 1629 | if is_pgo_mode(): |
| 1630 | logging.info('Still continuing because we are in PGO mode.') |
| 1631 | return True |
| 1632 | |
| 1633 | return False |
| 1634 | logging.info('Wait for idle CPU took %.1fs (utilization = %.3f).', |
| 1635 | time_passed, fraction_active_time) |
| 1636 | return True |
| 1637 | |
| 1638 | |
| 1639 | def log_process_activity(): |
| 1640 | """Logs the output of top. |
| 1641 | |
| 1642 | Useful to debug performance tests and to find runaway processes. |
| 1643 | """ |
| 1644 | logging.info('Logging current process activity using top and ps.') |
| 1645 | cmd = 'top -b -n1 -c' |
| 1646 | output = utils.run(cmd) |
| 1647 | logging.info(output) |
| 1648 | output = utils.run('ps axl') |
| 1649 | logging.info(output) |
| 1650 | |
| 1651 | |
| 1652 | def wait_for_cool_machine(): |
| 1653 | """ |
| 1654 | A simple heuristic to wait for a machine to cool. |
| 1655 | The code looks a bit 'magic', but we don't know ambient temperature |
| 1656 | nor machine characteristics and still would like to return the caller |
| 1657 | a machine that cooled down as much as reasonably possible. |
| 1658 | """ |
| 1659 | temperature = get_current_temperature_max() |
| 1660 | # We got here with a cold machine, return immediately. This should be the |
| 1661 | # most common case. |
| 1662 | if temperature < 50: |
| 1663 | return True |
| 1664 | logging.info('Got a hot machine of %dC. Sleeping 1 minute.', temperature) |
| 1665 | # A modest wait should cool the machine. |
| 1666 | time.sleep(60.0) |
| 1667 | temperature = get_current_temperature_max() |
| 1668 | # Atoms idle below 60 and everyone else should be even lower. |
| 1669 | if temperature < 62: |
| 1670 | return True |
| 1671 | # This should be rare. |
| 1672 | logging.info('Did not cool down (%dC). Sleeping 2 minutes.', temperature) |
| 1673 | time.sleep(120.0) |
| 1674 | temperature = get_current_temperature_max() |
| 1675 | # A temperature over 65'C doesn't give us much headroom to the critical |
| 1676 | # temperatures that start at 85'C (and PerfControl as of today will fail at |
| 1677 | # critical - 10'C). |
| 1678 | if temperature < 65: |
| 1679 | return True |
| 1680 | logging.warning('Did not cool down (%dC), giving up.', temperature) |
| 1681 | log_process_activity() |
| 1682 | return False |
| 1683 | |
| 1684 | |
| 1685 | # System paths for machine performance state. |
| 1686 | _CPUINFO = '/proc/cpuinfo' |
| 1687 | _DIRTY_WRITEBACK_CENTISECS = '/proc/sys/vm/dirty_writeback_centisecs' |
| 1688 | _KERNEL_MAX = '/sys/devices/system/cpu/kernel_max' |
| 1689 | _MEMINFO = '/proc/meminfo' |
| 1690 | _TEMP_SENSOR_RE = 'Reading temperature...([0-9]*)' |
| 1691 | |
Kristoffer Erlandsson | 1ad7db0 | 2017-11-01 11:28:44 +0100 | [diff] [blame] | 1692 | def _open_file(path): |
| 1693 | """ |
| 1694 | Opens a file and returns the file object. |
| 1695 | |
| 1696 | This method is intended to be mocked by tests. |
| 1697 | @return The open file object. |
| 1698 | """ |
| 1699 | return open(path) |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1700 | |
| 1701 | def _get_line_from_file(path, line): |
| 1702 | """ |
| 1703 | line can be an integer or |
| 1704 | line can be a string that matches the beginning of the line |
| 1705 | """ |
Kristoffer Erlandsson | 1ad7db0 | 2017-11-01 11:28:44 +0100 | [diff] [blame] | 1706 | with _open_file(path) as f: |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1707 | if isinstance(line, int): |
| 1708 | l = f.readline() |
| 1709 | for _ in range(0, line): |
| 1710 | l = f.readline() |
| 1711 | return l |
| 1712 | else: |
| 1713 | for l in f: |
| 1714 | if l.startswith(line): |
| 1715 | return l |
| 1716 | return None |
| 1717 | |
| 1718 | |
| 1719 | def _get_match_from_file(path, line, prefix, postfix): |
| 1720 | """ |
| 1721 | Matches line in path and returns string between first prefix and postfix. |
| 1722 | """ |
| 1723 | match = _get_line_from_file(path, line) |
| 1724 | # Strip everything from front of line including prefix. |
| 1725 | if prefix: |
| 1726 | match = re.split(prefix, match)[1] |
| 1727 | # Strip everything from back of string including first occurence of postfix. |
| 1728 | if postfix: |
| 1729 | match = re.split(postfix, match)[0] |
| 1730 | return match |
| 1731 | |
| 1732 | |
| 1733 | def _get_float_from_file(path, line, prefix, postfix): |
| 1734 | match = _get_match_from_file(path, line, prefix, postfix) |
| 1735 | return float(match) |
| 1736 | |
| 1737 | |
| 1738 | def _get_int_from_file(path, line, prefix, postfix): |
| 1739 | match = _get_match_from_file(path, line, prefix, postfix) |
| 1740 | return int(match) |
| 1741 | |
| 1742 | |
| 1743 | def _get_hex_from_file(path, line, prefix, postfix): |
| 1744 | match = _get_match_from_file(path, line, prefix, postfix) |
| 1745 | return int(match, 16) |
| 1746 | |
| 1747 | |
| 1748 | # The paths don't change. Avoid running find all the time. |
| 1749 | _hwmon_paths = None |
| 1750 | |
| 1751 | def _get_hwmon_paths(file_pattern): |
| 1752 | """ |
| 1753 | Returns a list of paths to the temperature sensors. |
| 1754 | """ |
| 1755 | # Some systems like daisy_spring only have the virtual hwmon. |
| 1756 | # And other systems like rambi only have coretemp.0. See crbug.com/360249. |
| 1757 | # /sys/class/hwmon/hwmon*/ |
| 1758 | # /sys/devices/virtual/hwmon/hwmon*/ |
| 1759 | # /sys/devices/platform/coretemp.0/ |
| 1760 | if not _hwmon_paths: |
| 1761 | cmd = 'find /sys/ -name "' + file_pattern + '"' |
| 1762 | _hwon_paths = utils.run(cmd, verbose=False).stdout.splitlines() |
| 1763 | return _hwon_paths |
| 1764 | |
| 1765 | |
| 1766 | def get_temperature_critical(): |
| 1767 | """ |
| 1768 | Returns temperature at which we will see some throttling in the system. |
| 1769 | """ |
| 1770 | min_temperature = 1000.0 |
| 1771 | paths = _get_hwmon_paths('temp*_crit') |
| 1772 | for path in paths: |
| 1773 | temperature = _get_float_from_file(path, 0, None, None) * 0.001 |
| 1774 | # Today typical for Intel is 98'C to 105'C while ARM is 85'C. Clamp to |
| 1775 | # the lowest known value. |
| 1776 | if (min_temperature < 60.0) or min_temperature > 150.0: |
| 1777 | logging.warning('Critical temperature of %.1fC was reset to 85.0C.', |
| 1778 | min_temperature) |
| 1779 | min_temperature = 85.0 |
| 1780 | |
| 1781 | min_temperature = min(temperature, min_temperature) |
| 1782 | return min_temperature |
| 1783 | |
| 1784 | |
| 1785 | def get_temperature_input_max(): |
| 1786 | """ |
| 1787 | Returns the maximum currently observed temperature. |
| 1788 | """ |
| 1789 | max_temperature = -1000.0 |
| 1790 | paths = _get_hwmon_paths('temp*_input') |
| 1791 | for path in paths: |
| 1792 | temperature = _get_float_from_file(path, 0, None, None) * 0.001 |
| 1793 | max_temperature = max(temperature, max_temperature) |
| 1794 | return max_temperature |
| 1795 | |
| 1796 | |
| 1797 | def get_thermal_zone_temperatures(): |
| 1798 | """ |
| 1799 | Returns the maximum currently observered temperature in thermal_zones. |
| 1800 | """ |
| 1801 | temperatures = [] |
| 1802 | for path in glob.glob('/sys/class/thermal/thermal_zone*/temp'): |
| 1803 | try: |
| 1804 | temperatures.append( |
| 1805 | _get_float_from_file(path, 0, None, None) * 0.001) |
| 1806 | except IOError: |
| 1807 | # Some devices (e.g. Veyron) may have reserved thermal zones that |
| 1808 | # are not active. Trying to read the temperature value would cause a |
| 1809 | # EINVAL IO error. |
| 1810 | continue |
| 1811 | return temperatures |
| 1812 | |
| 1813 | |
| 1814 | def get_ec_temperatures(): |
| 1815 | """ |
| 1816 | Uses ectool to return a list of all sensor temperatures in Celsius. |
Nicolas Boichat | eebd122 | 2017-11-10 10:21:42 -0800 | [diff] [blame] | 1817 | |
| 1818 | Output from ectool is either '0: 300' or '0: 300 K' (newer ectool |
| 1819 | includes the unit). |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1820 | """ |
| 1821 | temperatures = [] |
| 1822 | try: |
| 1823 | full_cmd = 'ectool temps all' |
| 1824 | lines = utils.run(full_cmd, verbose=False).stdout.splitlines() |
Nicolas Boichat | eebd122 | 2017-11-10 10:21:42 -0800 | [diff] [blame] | 1825 | pattern = re.compile('.*: (\d+)') |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1826 | for line in lines: |
Nicolas Boichat | eebd122 | 2017-11-10 10:21:42 -0800 | [diff] [blame] | 1827 | matched = pattern.match(line) |
| 1828 | temperature = int(matched.group(1)) - 273 |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 1829 | temperatures.append(temperature) |
| 1830 | except Exception: |
| 1831 | logging.warning('Unable to read temperature sensors using ectool.') |
| 1832 | for temperature in temperatures: |
| 1833 | # Sanity check for real world values. |
| 1834 | assert ((temperature > 10.0) and |
| 1835 | (temperature < 150.0)), ('Unreasonable temperature %.1fC.' % |
| 1836 | temperature) |
| 1837 | |
| 1838 | return temperatures |
| 1839 | |
| 1840 | |
| 1841 | def get_current_temperature_max(): |
| 1842 | """ |
| 1843 | Returns the highest reported board temperature (all sensors) in Celsius. |
| 1844 | """ |
| 1845 | temperature = max([get_temperature_input_max()] + |
| 1846 | get_thermal_zone_temperatures() + |
| 1847 | get_ec_temperatures()) |
| 1848 | # Sanity check for real world values. |
| 1849 | assert ((temperature > 10.0) and |
| 1850 | (temperature < 150.0)), ('Unreasonable temperature %.1fC.' % |
| 1851 | temperature) |
| 1852 | return temperature |
| 1853 | |
| 1854 | |
| 1855 | def get_cpu_cache_size(): |
| 1856 | """ |
| 1857 | Returns the last level CPU cache size in kBytes. |
| 1858 | """ |
| 1859 | cache_size = _get_int_from_file(_CPUINFO, 'cache size', ': ', ' KB') |
| 1860 | # Sanity check. |
| 1861 | assert cache_size >= 64, 'Unreasonably small cache.' |
| 1862 | return cache_size |
| 1863 | |
| 1864 | |
| 1865 | def get_cpu_model_frequency(): |
| 1866 | """ |
| 1867 | Returns the model frequency from the CPU model name on Intel only. This |
| 1868 | might be redundant with get_cpu_max_frequency. Unit is Hz. |
| 1869 | """ |
| 1870 | frequency = _get_float_from_file(_CPUINFO, 'model name', ' @ ', 'GHz') |
| 1871 | return 1.e9 * frequency |
| 1872 | |
| 1873 | |
| 1874 | def get_cpu_max_frequency(): |
| 1875 | """ |
| 1876 | Returns the largest of the max CPU core frequencies. The unit is Hz. |
| 1877 | """ |
| 1878 | max_frequency = -1 |
| 1879 | paths = _get_cpufreq_paths('cpuinfo_max_freq') |
| 1880 | for path in paths: |
| 1881 | # Convert from kHz to Hz. |
| 1882 | frequency = 1000 * _get_float_from_file(path, 0, None, None) |
| 1883 | max_frequency = max(frequency, max_frequency) |
| 1884 | # Sanity check. |
| 1885 | assert max_frequency > 1e8, 'Unreasonably low CPU frequency.' |
| 1886 | return max_frequency |
| 1887 | |
| 1888 | |
| 1889 | def get_cpu_min_frequency(): |
| 1890 | """ |
| 1891 | Returns the smallest of the minimum CPU core frequencies. |
| 1892 | """ |
| 1893 | min_frequency = 1e20 |
| 1894 | paths = _get_cpufreq_paths('cpuinfo_min_freq') |
| 1895 | for path in paths: |
| 1896 | frequency = _get_float_from_file(path, 0, None, None) |
| 1897 | min_frequency = min(frequency, min_frequency) |
| 1898 | # Sanity check. |
| 1899 | assert min_frequency > 1e8, 'Unreasonably low CPU frequency.' |
| 1900 | return min_frequency |
| 1901 | |
| 1902 | |
| 1903 | def get_cpu_model(): |
| 1904 | """ |
| 1905 | Returns the CPU model. |
| 1906 | Only works on Intel. |
| 1907 | """ |
| 1908 | cpu_model = _get_int_from_file(_CPUINFO, 'model\t', ': ', None) |
| 1909 | return cpu_model |
| 1910 | |
| 1911 | |
| 1912 | def get_cpu_family(): |
| 1913 | """ |
| 1914 | Returns the CPU family. |
| 1915 | Only works on Intel. |
| 1916 | """ |
| 1917 | cpu_family = _get_int_from_file(_CPUINFO, 'cpu family\t', ': ', None) |
| 1918 | return cpu_family |
| 1919 | |
| 1920 | |
| 1921 | def get_board_property(key): |
| 1922 | """ |
| 1923 | Get a specific property from /etc/lsb-release. |
| 1924 | |
| 1925 | @param key: board property to return value for |
| 1926 | |
| 1927 | @return the value or '' if not present |
| 1928 | """ |
| 1929 | with open('/etc/lsb-release') as f: |
| 1930 | pattern = '%s=(.*)' % key |
| 1931 | pat = re.search(pattern, f.read()) |
| 1932 | if pat: |
| 1933 | return pat.group(1) |
| 1934 | return '' |
| 1935 | |
| 1936 | |
| 1937 | def get_board(): |
| 1938 | """ |
| 1939 | Get the ChromeOS release board name from /etc/lsb-release. |
| 1940 | """ |
| 1941 | return get_board_property('BOARD') |
| 1942 | |
| 1943 | |
| 1944 | def get_board_type(): |
| 1945 | """ |
| 1946 | Get the ChromeOS board type from /etc/lsb-release. |
| 1947 | |
| 1948 | @return device type. |
| 1949 | """ |
| 1950 | return get_board_property('DEVICETYPE') |
| 1951 | |
| 1952 | |
Puthikorn Voravootivat | 0f6d230 | 2017-11-27 16:31:00 -0800 | [diff] [blame] | 1953 | def get_ec_version(): |
| 1954 | """Get the ec version as strings. |
| 1955 | |
| 1956 | @returns a string representing this host's ec version. |
| 1957 | """ |
Puthikorn Voravootivat | 65ad767 | 2018-01-30 14:00:01 -0800 | [diff] [blame] | 1958 | command = 'mosys ec info -s fw_version' |
Puthikorn Voravootivat | 720640d | 2018-02-16 17:32:38 -0800 | [diff] [blame] | 1959 | result = utils.run(command, ignore_status=True) |
| 1960 | if result.exit_status != 0: |
| 1961 | return '' |
| 1962 | return result.stdout.strip() |
Puthikorn Voravootivat | 0f6d230 | 2017-11-27 16:31:00 -0800 | [diff] [blame] | 1963 | |
| 1964 | |
| 1965 | def get_firmware_version(): |
| 1966 | """Get the firmware version as strings. |
| 1967 | |
| 1968 | @returns a string representing this host's firmware version. |
| 1969 | """ |
| 1970 | return utils.run('crossystem fwid').stdout.strip() |
| 1971 | |
| 1972 | |
| 1973 | def get_hardware_revision(): |
| 1974 | """Get the hardware revision as strings. |
| 1975 | |
| 1976 | @returns a string representing this host's hardware revision. |
| 1977 | """ |
Puthikorn Voravootivat | 65ad767 | 2018-01-30 14:00:01 -0800 | [diff] [blame] | 1978 | command = 'mosys platform version' |
Puthikorn Voravootivat | 720640d | 2018-02-16 17:32:38 -0800 | [diff] [blame] | 1979 | result = utils.run(command, ignore_status=True) |
| 1980 | if result.exit_status != 0: |
| 1981 | return '' |
| 1982 | return result.stdout.strip() |
Puthikorn Voravootivat | 0f6d230 | 2017-11-27 16:31:00 -0800 | [diff] [blame] | 1983 | |
| 1984 | |
| 1985 | def get_kernel_version(): |
| 1986 | """Get the kernel version as strings. |
| 1987 | |
| 1988 | @returns a string representing this host's kernel version. |
| 1989 | """ |
| 1990 | return utils.run('uname -r').stdout.strip() |
| 1991 | |
| 1992 | |
Puthikorn Voravootivat | a83e82c | 2018-02-15 17:49:45 -0800 | [diff] [blame] | 1993 | def get_cpu_name(): |
| 1994 | """Get the cpu name as strings. |
| 1995 | |
| 1996 | @returns a string representing this host's cpu name. |
| 1997 | """ |
| 1998 | |
| 1999 | # Try get cpu name from device tree first |
| 2000 | if os.path.exists("/proc/device-tree/compatible"): |
| 2001 | command = "sed -e 's/\\x0/\\n/g' /proc/device-tree/compatible | tail -1" |
| 2002 | return utils.run(command).stdout.strip().replace(',', ' ') |
| 2003 | |
| 2004 | |
| 2005 | # Get cpu name from uname -p |
| 2006 | command = "uname -p" |
| 2007 | ret = utils.run(command).stdout.strip() |
| 2008 | |
| 2009 | # 'uname -p' return variant of unknown or amd64 or x86_64 or i686 |
| 2010 | # Try get cpu name from /proc/cpuinfo instead |
| 2011 | if re.match("unknown|amd64|[ix][0-9]?86(_64)?", ret, re.IGNORECASE): |
| 2012 | command = "grep model.name /proc/cpuinfo | cut -f 2 -d: | head -1" |
| 2013 | ret = utils.run(command).stdout.strip() |
| 2014 | |
| 2015 | # Remove bloat from CPU name, for example |
| 2016 | # 'Intel(R) Core(TM) i5-7Y57 CPU @ 1.20GHz' -> 'Intel Core i5-7Y57' |
| 2017 | # 'Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz' -> 'Intel Xeon E5-2690 v4' |
| 2018 | # 'AMD A10-7850K APU with Radeon(TM) R7 Graphics' -> 'AMD A10-7850K' |
| 2019 | # 'AMD GX-212JC SOC with Radeon(TM) R2E Graphics' -> 'AMD GX-212JC' |
| 2020 | trim_re = " (@|processor|apu|soc|radeon).*|\(.*?\)| cpu" |
| 2021 | return re.sub(trim_re, '', ret, flags=re.IGNORECASE) |
| 2022 | |
| 2023 | |
| 2024 | def get_screen_resolution(): |
| 2025 | """Get the screen(s) resolution as strings. |
| 2026 | In case of more than 1 monitor, return resolution for each monitor separate |
| 2027 | with plus sign. |
| 2028 | |
| 2029 | @returns a string representing this host's screen(s) resolution. |
| 2030 | """ |
| 2031 | command = 'for f in /sys/class/drm/*/*/modes; do head -1 $f; done' |
| 2032 | ret = utils.run(command, ignore_status=True) |
| 2033 | # We might have Chromebox without a screen |
| 2034 | if ret.exit_status != 0: |
| 2035 | return '' |
| 2036 | return ret.stdout.strip().replace('\n', '+') |
| 2037 | |
| 2038 | |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 2039 | def get_board_with_frequency_and_memory(): |
| 2040 | """ |
| 2041 | Returns a board name modified with CPU frequency and memory size to |
| 2042 | differentiate between different board variants. For instance |
| 2043 | link -> link_1.8GHz_4GB. |
| 2044 | """ |
| 2045 | board_name = get_board() |
| 2046 | if is_virtual_machine(): |
| 2047 | board = '%s_VM' % board_name |
| 2048 | else: |
Puthikorn Voravootivat | a83e82c | 2018-02-15 17:49:45 -0800 | [diff] [blame] | 2049 | memory = get_mem_total_gb() |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 2050 | # Convert frequency to GHz with 1 digit accuracy after the |
| 2051 | # decimal point. |
| 2052 | frequency = int(round(get_cpu_max_frequency() * 1e-8)) * 0.1 |
| 2053 | board = '%s_%1.1fGHz_%dGB' % (board_name, frequency, memory) |
| 2054 | return board |
| 2055 | |
| 2056 | |
| 2057 | def get_mem_total(): |
| 2058 | """ |
| 2059 | Returns the total memory available in the system in MBytes. |
| 2060 | """ |
| 2061 | mem_total = _get_float_from_file(_MEMINFO, 'MemTotal:', 'MemTotal:', ' kB') |
| 2062 | # Sanity check, all Chromebooks have at least 1GB of memory. |
| 2063 | assert mem_total > 256 * 1024, 'Unreasonable amount of memory.' |
| 2064 | return mem_total / 1024 |
| 2065 | |
| 2066 | |
Puthikorn Voravootivat | a83e82c | 2018-02-15 17:49:45 -0800 | [diff] [blame] | 2067 | def get_mem_total_gb(): |
| 2068 | """ |
| 2069 | Returns the total memory available in the system in GBytes. |
| 2070 | """ |
| 2071 | return int(round(get_mem_total() / 1024.0)) |
| 2072 | |
| 2073 | |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 2074 | def get_mem_free(): |
| 2075 | """ |
| 2076 | Returns the currently free memory in the system in MBytes. |
| 2077 | """ |
| 2078 | mem_free = _get_float_from_file(_MEMINFO, 'MemFree:', 'MemFree:', ' kB') |
| 2079 | return mem_free / 1024 |
| 2080 | |
Kristoffer Erlandsson | b84da2b | 2017-11-01 21:29:24 +0100 | [diff] [blame] | 2081 | def get_mem_free_plus_buffers_and_cached(): |
| 2082 | """ |
| 2083 | Returns the free memory in MBytes, counting buffers and cached as free. |
| 2084 | |
| 2085 | This is most often the most interesting number since buffers and cached |
| 2086 | memory can be reclaimed on demand. Note however, that there are cases |
| 2087 | where this as misleading as well, for example used tmpfs space |
| 2088 | count as Cached but can not be reclaimed on demand. |
| 2089 | See https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt. |
| 2090 | """ |
| 2091 | free_mb = get_mem_free() |
| 2092 | cached_mb = (_get_float_from_file( |
| 2093 | _MEMINFO, 'Cached:', 'Cached:', ' kB') / 1024) |
| 2094 | buffers_mb = (_get_float_from_file( |
| 2095 | _MEMINFO, 'Buffers:', 'Buffers:', ' kB') / 1024) |
| 2096 | return free_mb + buffers_mb + cached_mb |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 2097 | |
| 2098 | def get_kernel_max(): |
| 2099 | """ |
| 2100 | Returns content of kernel_max. |
| 2101 | """ |
| 2102 | kernel_max = _get_int_from_file(_KERNEL_MAX, 0, None, None) |
| 2103 | # Sanity check. |
| 2104 | assert ((kernel_max > 0) and (kernel_max < 257)), 'Unreasonable kernel_max.' |
| 2105 | return kernel_max |
| 2106 | |
| 2107 | |
| 2108 | def set_high_performance_mode(): |
| 2109 | """ |
| 2110 | Sets the kernel governor mode to the highest setting. |
| 2111 | Returns previous governor state. |
| 2112 | """ |
| 2113 | original_governors = get_scaling_governor_states() |
| 2114 | set_scaling_governors('performance') |
| 2115 | return original_governors |
| 2116 | |
| 2117 | |
| 2118 | def set_scaling_governors(value): |
| 2119 | """ |
| 2120 | Sets all scaling governor to string value. |
| 2121 | Sample values: 'performance', 'interactive', 'ondemand', 'powersave'. |
| 2122 | """ |
| 2123 | paths = _get_cpufreq_paths('scaling_governor') |
| 2124 | for path in paths: |
| 2125 | cmd = 'echo %s > %s' % (value, path) |
| 2126 | logging.info('Writing scaling governor mode \'%s\' -> %s', value, path) |
| 2127 | # On Tegra CPUs can be dynamically enabled/disabled. Ignore failures. |
| 2128 | utils.system(cmd, ignore_status=True) |
| 2129 | |
| 2130 | |
| 2131 | def _get_cpufreq_paths(filename): |
| 2132 | """ |
| 2133 | Returns a list of paths to the governors. |
| 2134 | """ |
| 2135 | cmd = 'ls /sys/devices/system/cpu/cpu*/cpufreq/' + filename |
| 2136 | paths = utils.run(cmd, verbose=False).stdout.splitlines() |
| 2137 | return paths |
| 2138 | |
| 2139 | |
| 2140 | def get_scaling_governor_states(): |
| 2141 | """ |
| 2142 | Returns a list of (performance governor path, current state) tuples. |
| 2143 | """ |
| 2144 | paths = _get_cpufreq_paths('scaling_governor') |
| 2145 | path_value_list = [] |
| 2146 | for path in paths: |
| 2147 | value = _get_line_from_file(path, 0) |
| 2148 | path_value_list.append((path, value)) |
| 2149 | return path_value_list |
| 2150 | |
| 2151 | |
| 2152 | def restore_scaling_governor_states(path_value_list): |
| 2153 | """ |
| 2154 | Restores governor states. Inverse operation to get_scaling_governor_states. |
| 2155 | """ |
| 2156 | for (path, value) in path_value_list: |
| 2157 | cmd = 'echo %s > %s' % (value.rstrip('\n'), path) |
| 2158 | # On Tegra CPUs can be dynamically enabled/disabled. Ignore failures. |
| 2159 | utils.system(cmd, ignore_status=True) |
| 2160 | |
| 2161 | |
| 2162 | def get_dirty_writeback_centisecs(): |
| 2163 | """ |
| 2164 | Reads /proc/sys/vm/dirty_writeback_centisecs. |
| 2165 | """ |
| 2166 | time = _get_int_from_file(_DIRTY_WRITEBACK_CENTISECS, 0, None, None) |
| 2167 | return time |
| 2168 | |
| 2169 | |
| 2170 | def set_dirty_writeback_centisecs(time=60000): |
| 2171 | """ |
| 2172 | In hundredths of a second, this is how often pdflush wakes up to write data |
| 2173 | to disk. The default wakes up the two (or more) active threads every five |
| 2174 | seconds. The ChromeOS default is 10 minutes. |
| 2175 | |
| 2176 | We use this to set as low as 1 second to flush error messages in system |
| 2177 | logs earlier to disk. |
| 2178 | """ |
| 2179 | # Flush buffers first to make this function synchronous. |
| 2180 | utils.system('sync') |
| 2181 | if time >= 0: |
| 2182 | cmd = 'echo %d > %s' % (time, _DIRTY_WRITEBACK_CENTISECS) |
| 2183 | utils.system(cmd) |
| 2184 | |
| 2185 | |
| 2186 | def wflinfo_cmd(): |
| 2187 | """ |
| 2188 | Returns a wflinfo command appropriate to the current graphics platform/api. |
| 2189 | """ |
| 2190 | return 'wflinfo -p %s -a %s' % (graphics_platform(), graphics_api()) |
| 2191 | |
| 2192 | |
| 2193 | def has_mali(): |
| 2194 | """ @return: True if system has a Mali GPU enabled.""" |
| 2195 | return os.path.exists('/dev/mali0') |
| 2196 | |
| 2197 | def get_gpu_family(): |
| 2198 | """Returns the GPU family name.""" |
| 2199 | global pciid_to_amd_architecture |
| 2200 | global pciid_to_intel_architecture |
| 2201 | |
| 2202 | socfamily = get_cpu_soc_family() |
| 2203 | if socfamily == 'exynos5' or socfamily == 'rockchip' or has_mali(): |
| 2204 | cmd = wflinfo_cmd() |
| 2205 | wflinfo = utils.system_output(cmd, |
| 2206 | retain_output=True, |
| 2207 | ignore_status=False) |
| 2208 | version = re.findall(r'OpenGL renderer string: ' |
| 2209 | r'Mali-T([0-9]+)', wflinfo) |
| 2210 | if version: |
| 2211 | return 'mali-t%s' % version[0] |
| 2212 | return 'mali-unrecognized' |
| 2213 | if socfamily == 'tegra': |
| 2214 | return 'tegra' |
| 2215 | if os.path.exists('/sys/kernel/debug/pvr'): |
| 2216 | return 'rogue' |
| 2217 | |
| 2218 | pci_vga_device = utils.run("lspci | grep VGA").stdout.rstrip('\n') |
| 2219 | bus_device_function = pci_vga_device.partition(' ')[0] |
| 2220 | pci_path = '/sys/bus/pci/devices/0000:' + bus_device_function + '/device' |
| 2221 | |
| 2222 | if not os.path.exists(pci_path): |
| 2223 | raise error.TestError('PCI device 0000:' + bus_device_function + ' not found') |
| 2224 | |
| 2225 | device_id = utils.read_one_line(pci_path).lower() |
| 2226 | |
| 2227 | if "Advanced Micro Devices" in pci_vga_device: |
| 2228 | if not pciid_to_amd_architecture: |
| 2229 | with open(_AMD_PCI_IDS_FILE_PATH, 'r') as in_f: |
| 2230 | pciid_to_amd_architecture = json.load(in_f) |
| 2231 | |
| 2232 | return pciid_to_amd_architecture[device_id] |
| 2233 | |
| 2234 | if "Intel Corporation" in pci_vga_device: |
| 2235 | # Only load Intel PCI ID file once and only if necessary. |
| 2236 | if not pciid_to_intel_architecture: |
| 2237 | with open(_INTEL_PCI_IDS_FILE_PATH, 'r') as in_f: |
| 2238 | pciid_to_intel_architecture = json.load(in_f) |
| 2239 | |
| 2240 | return pciid_to_intel_architecture[device_id] |
| 2241 | |
| 2242 | # TODO(ihf): Consider using /etc/lsb-release DEVICETYPE != CHROMEBOOK/CHROMEBASE |
| 2243 | # for sanity check, but usage seems a bit inconsistent. See |
| 2244 | # src/third_party/chromiumos-overlay/eclass/appid.eclass |
| 2245 | _BOARDS_WITHOUT_MONITOR = [ |
| 2246 | 'anglar', 'mccloud', 'monroe', 'ninja', 'rikku', 'guado', 'jecht', 'tidus', |
Brian Norris | bb74538 | 2017-06-30 11:49:27 -0700 | [diff] [blame] | 2247 | 'beltino', 'panther', 'stumpy', 'panther', 'tricky', 'zako', 'veyron_rialto' |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 2248 | ] |
| 2249 | |
| 2250 | |
| 2251 | def has_no_monitor(): |
| 2252 | """Returns whether a machine doesn't have a built-in monitor.""" |
| 2253 | board_name = get_board() |
| 2254 | if board_name in _BOARDS_WITHOUT_MONITOR: |
| 2255 | return True |
| 2256 | |
| 2257 | return False |
| 2258 | |
| 2259 | |
| 2260 | def get_fixed_dst_drive(): |
| 2261 | """ |
| 2262 | Return device name for internal disk. |
| 2263 | Example: return /dev/sda for falco booted from usb |
| 2264 | """ |
| 2265 | cmd = ' '.join(['. /usr/sbin/write_gpt.sh;', |
| 2266 | '. /usr/share/misc/chromeos-common.sh;', |
| 2267 | 'load_base_vars;', |
| 2268 | 'get_fixed_dst_drive']) |
| 2269 | return utils.system_output(cmd) |
| 2270 | |
| 2271 | |
| 2272 | def get_root_device(): |
| 2273 | """ |
| 2274 | Return root device. |
| 2275 | Will return correct disk device even system boot from /dev/dm-0 |
| 2276 | Example: return /dev/sdb for falco booted from usb |
| 2277 | """ |
| 2278 | return utils.system_output('rootdev -s -d') |
| 2279 | |
| 2280 | |
| 2281 | def get_root_partition(): |
| 2282 | """ |
| 2283 | Return current root partition |
| 2284 | Example: return /dev/sdb3 for falco booted from usb |
| 2285 | """ |
| 2286 | return utils.system_output('rootdev -s') |
| 2287 | |
| 2288 | |
| 2289 | def get_free_root_partition(root_part=None): |
| 2290 | """ |
| 2291 | Return currently unused root partion |
| 2292 | Example: return /dev/sdb5 for falco booted from usb |
| 2293 | |
| 2294 | @param root_part: cuurent root partition |
| 2295 | """ |
| 2296 | spare_root_map = {'3': '5', '5': '3'} |
| 2297 | if not root_part: |
| 2298 | root_part = get_root_partition() |
| 2299 | return root_part[:-1] + spare_root_map[root_part[-1]] |
| 2300 | |
| 2301 | |
| 2302 | def get_kernel_partition(root_part=None): |
| 2303 | """ |
| 2304 | Return current kernel partition |
| 2305 | Example: return /dev/sda2 for falco booted from usb |
| 2306 | |
| 2307 | @param root_part: current root partition |
| 2308 | """ |
| 2309 | if not root_part: |
| 2310 | root_part = get_root_partition() |
| 2311 | current_kernel_map = {'3': '2', '5': '4'} |
| 2312 | return root_part[:-1] + current_kernel_map[root_part[-1]] |
| 2313 | |
| 2314 | |
| 2315 | def get_free_kernel_partition(root_part=None): |
| 2316 | """ |
| 2317 | return currently unused kernel partition |
| 2318 | Example: return /dev/sda4 for falco booted from usb |
| 2319 | |
| 2320 | @param root_part: current root partition |
| 2321 | """ |
| 2322 | kernel_part = get_kernel_partition(root_part) |
| 2323 | spare_kernel_map = {'2': '4', '4': '2'} |
| 2324 | return kernel_part[:-1] + spare_kernel_map[kernel_part[-1]] |
| 2325 | |
| 2326 | |
| 2327 | def is_booted_from_internal_disk(): |
| 2328 | """Return True if boot from internal disk. False, otherwise.""" |
| 2329 | return get_root_device() == get_fixed_dst_drive() |
| 2330 | |
| 2331 | |
| 2332 | def get_ui_use_flags(): |
| 2333 | """Parses the USE flags as listed in /etc/ui_use_flags.txt. |
| 2334 | |
| 2335 | @return: A list of flag strings found in the ui use flags file. |
| 2336 | """ |
| 2337 | flags = [] |
| 2338 | for flag in utils.read_file(_UI_USE_FLAGS_FILE_PATH).splitlines(): |
| 2339 | # Removes everything after the '#'. |
| 2340 | flag_before_comment = flag.split('#')[0].strip() |
| 2341 | if len(flag_before_comment) != 0: |
| 2342 | flags.append(flag_before_comment) |
| 2343 | |
| 2344 | return flags |
| 2345 | |
| 2346 | |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 2347 | def graphics_platform(): |
| 2348 | """ |
| 2349 | Return a string identifying the graphics platform, |
| 2350 | e.g. 'glx' or 'x11_egl' or 'gbm' |
| 2351 | """ |
Po-Hsien Wang | dab8e18 | 2017-05-03 10:25:41 -0700 | [diff] [blame] | 2352 | return 'null' |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 2353 | |
| 2354 | |
| 2355 | def graphics_api(): |
| 2356 | """Return a string identifying the graphics api, e.g. gl or gles2.""" |
| 2357 | use_flags = get_ui_use_flags() |
| 2358 | if 'opengles' in use_flags: |
| 2359 | return 'gles2' |
| 2360 | return 'gl' |
| 2361 | |
| 2362 | |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 2363 | def is_vm(): |
| 2364 | """Check if the process is running in a virtual machine. |
| 2365 | |
| 2366 | @return: True if the process is running in a virtual machine, otherwise |
| 2367 | return False. |
| 2368 | """ |
| 2369 | try: |
| 2370 | virt = utils.run('sudo -n virt-what').stdout.strip() |
| 2371 | logging.debug('virt-what output: %s', virt) |
| 2372 | return bool(virt) |
| 2373 | except error.CmdError: |
| 2374 | logging.warn('Package virt-what is not installed, default to assume ' |
| 2375 | 'it is not a virtual machine.') |
| 2376 | return False |
| 2377 | |
| 2378 | |
| 2379 | def is_package_installed(package): |
| 2380 | """Check if a package is installed already. |
| 2381 | |
| 2382 | @return: True if the package is already installed, otherwise return False. |
| 2383 | """ |
| 2384 | try: |
| 2385 | utils.run(_CHECK_PACKAGE_INSTALLED_COMMAND % package) |
| 2386 | return True |
| 2387 | except error.CmdError: |
| 2388 | logging.warn('Package %s is not installed.', package) |
| 2389 | return False |
| 2390 | |
| 2391 | |
| 2392 | def is_python_package_installed(package): |
| 2393 | """Check if a Python package is installed already. |
| 2394 | |
| 2395 | @return: True if the package is already installed, otherwise return False. |
| 2396 | """ |
| 2397 | try: |
| 2398 | __import__(package) |
| 2399 | return True |
| 2400 | except ImportError: |
| 2401 | logging.warn('Python package %s is not installed.', package) |
| 2402 | return False |
| 2403 | |
| 2404 | |
| 2405 | def run_sql_cmd(server, user, password, command, database=''): |
| 2406 | """Run the given sql command against the specified database. |
| 2407 | |
| 2408 | @param server: Hostname or IP address of the MySQL server. |
| 2409 | @param user: User name to log in the MySQL server. |
| 2410 | @param password: Password to log in the MySQL server. |
| 2411 | @param command: SQL command to run. |
| 2412 | @param database: Name of the database to run the command. Default to empty |
| 2413 | for command that does not require specifying database. |
| 2414 | |
| 2415 | @return: The stdout of the command line. |
| 2416 | """ |
| 2417 | cmd = ('mysql -u%s -p%s --host %s %s -e "%s"' % |
| 2418 | (user, password, server, database, command)) |
| 2419 | # Set verbose to False so the command line won't be logged, as it includes |
| 2420 | # database credential. |
Aviv Keshet | b395044 | 2018-01-24 15:32:03 -0800 | [diff] [blame] | 2421 | return utils.run(cmd, verbose=False).stdout |