Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame^] | 1 | #!/usr/bin/env python |
| 2 | # |
| 3 | # Copyright 2013 The Chromium Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | |
| 7 | """A class to keep track of devices across builds and report state.""" |
| 8 | |
| 9 | import argparse |
| 10 | import json |
| 11 | import logging |
| 12 | import os |
| 13 | import psutil |
| 14 | import re |
| 15 | import signal |
| 16 | import sys |
| 17 | |
| 18 | sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) |
| 19 | import devil_chromium |
| 20 | from devil import devil_env |
| 21 | from devil.android import battery_utils |
| 22 | from devil.android import device_blacklist |
| 23 | from devil.android import device_errors |
| 24 | from devil.android import device_list |
| 25 | from devil.android import device_utils |
| 26 | from devil.android.sdk import adb_wrapper |
| 27 | from devil.constants import exit_codes |
| 28 | from devil.utils import lsusb |
| 29 | from devil.utils import reset_usb |
| 30 | from devil.utils import run_tests_helper |
| 31 | from pylib.constants import host_paths |
| 32 | |
| 33 | _RE_DEVICE_ID = re.compile(r'Device ID = (\d+)') |
| 34 | |
| 35 | |
| 36 | def KillAllAdb(): |
| 37 | def GetAllAdb(): |
| 38 | for p in psutil.process_iter(): |
| 39 | try: |
| 40 | if 'adb' in p.name: |
| 41 | yield p |
| 42 | except (psutil.NoSuchProcess, psutil.AccessDenied): |
| 43 | pass |
| 44 | |
| 45 | for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]: |
| 46 | for p in GetAllAdb(): |
| 47 | try: |
| 48 | logging.info('kill %d %d (%s [%s])', sig, p.pid, p.name, |
| 49 | ' '.join(p.cmdline)) |
| 50 | p.send_signal(sig) |
| 51 | except (psutil.NoSuchProcess, psutil.AccessDenied): |
| 52 | pass |
| 53 | for p in GetAllAdb(): |
| 54 | try: |
| 55 | logging.error('Unable to kill %d (%s [%s])', p.pid, p.name, |
| 56 | ' '.join(p.cmdline)) |
| 57 | except (psutil.NoSuchProcess, psutil.AccessDenied): |
| 58 | pass |
| 59 | |
| 60 | |
| 61 | def _IsBlacklisted(serial, blacklist): |
| 62 | return blacklist and serial in blacklist.Read() |
| 63 | |
| 64 | |
| 65 | def _BatteryStatus(device, blacklist): |
| 66 | battery_info = {} |
| 67 | try: |
| 68 | battery = battery_utils.BatteryUtils(device) |
| 69 | battery_info = battery.GetBatteryInfo(timeout=5) |
| 70 | battery_level = int(battery_info.get('level', 100)) |
| 71 | |
| 72 | if battery_level < 15: |
| 73 | logging.error('Critically low battery level (%d)', battery_level) |
| 74 | battery = battery_utils.BatteryUtils(device) |
| 75 | if not battery.GetCharging(): |
| 76 | battery.SetCharging(True) |
| 77 | if blacklist: |
| 78 | blacklist.Extend([device.adb.GetDeviceSerial()], reason='low_battery') |
| 79 | |
| 80 | except device_errors.CommandFailedError: |
| 81 | logging.exception('Failed to get battery information for %s', |
| 82 | str(device)) |
| 83 | |
| 84 | return battery_info |
| 85 | |
| 86 | |
| 87 | def _IMEISlice(device): |
| 88 | imei_slice = '' |
| 89 | try: |
| 90 | for l in device.RunShellCommand(['dumpsys', 'iphonesubinfo'], |
| 91 | check_return=True, timeout=5): |
| 92 | m = _RE_DEVICE_ID.match(l) |
| 93 | if m: |
| 94 | imei_slice = m.group(1)[-6:] |
| 95 | except device_errors.CommandFailedError: |
| 96 | logging.exception('Failed to get IMEI slice for %s', str(device)) |
| 97 | |
| 98 | return imei_slice |
| 99 | |
| 100 | |
| 101 | def DeviceStatus(devices, blacklist): |
| 102 | """Generates status information for the given devices. |
| 103 | |
| 104 | Args: |
| 105 | devices: The devices to generate status for. |
| 106 | blacklist: The current device blacklist. |
| 107 | Returns: |
| 108 | A dict of the following form: |
| 109 | { |
| 110 | '<serial>': { |
| 111 | 'serial': '<serial>', |
| 112 | 'adb_status': str, |
| 113 | 'usb_status': bool, |
| 114 | 'blacklisted': bool, |
| 115 | # only if the device is connected and not blacklisted |
| 116 | 'type': ro.build.product, |
| 117 | 'build': ro.build.id, |
| 118 | 'build_detail': ro.build.fingerprint, |
| 119 | 'battery': { |
| 120 | ... |
| 121 | }, |
| 122 | 'imei_slice': str, |
| 123 | 'wifi_ip': str, |
| 124 | }, |
| 125 | ... |
| 126 | } |
| 127 | """ |
| 128 | adb_devices = { |
| 129 | a[0].GetDeviceSerial(): a |
| 130 | for a in adb_wrapper.AdbWrapper.Devices(desired_state=None, long_list=True) |
| 131 | } |
| 132 | usb_devices = set(lsusb.get_android_devices()) |
| 133 | |
| 134 | def blacklisting_device_status(device): |
| 135 | serial = device.adb.GetDeviceSerial() |
| 136 | adb_status = ( |
| 137 | adb_devices[serial][1] if serial in adb_devices |
| 138 | else 'missing') |
| 139 | usb_status = bool(serial in usb_devices) |
| 140 | |
| 141 | device_status = { |
| 142 | 'serial': serial, |
| 143 | 'adb_status': adb_status, |
| 144 | 'usb_status': usb_status, |
| 145 | } |
| 146 | |
| 147 | if not _IsBlacklisted(serial, blacklist): |
| 148 | if adb_status == 'device': |
| 149 | try: |
| 150 | build_product = device.build_product |
| 151 | build_id = device.build_id |
| 152 | build_fingerprint = device.GetProp('ro.build.fingerprint', cache=True) |
| 153 | wifi_ip = device.GetProp('dhcp.wlan0.ipaddress') |
| 154 | battery_info = _BatteryStatus(device, blacklist) |
| 155 | imei_slice = _IMEISlice(device) |
| 156 | |
| 157 | if (device.product_name == 'mantaray' and |
| 158 | battery_info.get('AC powered', None) != 'true'): |
| 159 | logging.error('Mantaray device not connected to AC power.') |
| 160 | |
| 161 | device_status.update({ |
| 162 | 'ro.build.product': build_product, |
| 163 | 'ro.build.id': build_id, |
| 164 | 'ro.build.fingerprint': build_fingerprint, |
| 165 | 'battery': battery_info, |
| 166 | 'imei_slice': imei_slice, |
| 167 | 'wifi_ip': wifi_ip, |
| 168 | |
| 169 | # TODO(jbudorick): Remove these once no clients depend on them. |
| 170 | 'type': build_product, |
| 171 | 'build': build_id, |
| 172 | 'build_detail': build_fingerprint, |
| 173 | }) |
| 174 | |
| 175 | except device_errors.CommandFailedError: |
| 176 | logging.exception('Failure while getting device status for %s.', |
| 177 | str(device)) |
| 178 | if blacklist: |
| 179 | blacklist.Extend([serial], reason='status_check_failure') |
| 180 | |
| 181 | except device_errors.CommandTimeoutError: |
| 182 | logging.exception('Timeout while getting device status for %s.', |
| 183 | str(device)) |
| 184 | if blacklist: |
| 185 | blacklist.Extend([serial], reason='status_check_timeout') |
| 186 | |
| 187 | elif blacklist: |
| 188 | blacklist.Extend([serial], |
| 189 | reason=adb_status if usb_status else 'offline') |
| 190 | |
| 191 | device_status['blacklisted'] = _IsBlacklisted(serial, blacklist) |
| 192 | |
| 193 | return device_status |
| 194 | |
| 195 | parallel_devices = device_utils.DeviceUtils.parallel(devices) |
| 196 | statuses = parallel_devices.pMap(blacklisting_device_status).pGet(None) |
| 197 | return statuses |
| 198 | |
| 199 | |
| 200 | def RecoverDevices(devices, blacklist): |
| 201 | """Attempts to recover any inoperable devices in the provided list. |
| 202 | |
| 203 | Args: |
| 204 | devices: The list of devices to attempt to recover. |
| 205 | blacklist: The current device blacklist, which will be used then |
| 206 | reset. |
| 207 | Returns: |
| 208 | Nothing. |
| 209 | """ |
| 210 | |
| 211 | statuses = DeviceStatus(devices, blacklist) |
| 212 | |
| 213 | should_restart_usb = set( |
| 214 | status['serial'] for status in statuses |
| 215 | if (not status['usb_status'] |
| 216 | or status['adb_status'] in ('offline', 'missing'))) |
| 217 | should_restart_adb = should_restart_usb.union(set( |
| 218 | status['serial'] for status in statuses |
| 219 | if status['adb_status'] == 'unauthorized')) |
| 220 | should_reboot_device = should_restart_adb.union(set( |
| 221 | status['serial'] for status in statuses |
| 222 | if status['blacklisted'])) |
| 223 | |
| 224 | logging.debug('Should restart USB for:') |
| 225 | for d in should_restart_usb: |
| 226 | logging.debug(' %s', d) |
| 227 | logging.debug('Should restart ADB for:') |
| 228 | for d in should_restart_adb: |
| 229 | logging.debug(' %s', d) |
| 230 | logging.debug('Should reboot:') |
| 231 | for d in should_reboot_device: |
| 232 | logging.debug(' %s', d) |
| 233 | |
| 234 | if blacklist: |
| 235 | blacklist.Reset() |
| 236 | |
| 237 | if should_restart_adb: |
| 238 | KillAllAdb() |
| 239 | for serial in should_restart_usb: |
| 240 | try: |
| 241 | reset_usb.reset_android_usb(serial) |
| 242 | except IOError: |
| 243 | logging.exception('Unable to reset USB for %s.', serial) |
| 244 | if blacklist: |
| 245 | blacklist.Extend([serial], reason='usb_failure') |
| 246 | except device_errors.DeviceUnreachableError: |
| 247 | logging.exception('Unable to reset USB for %s.', serial) |
| 248 | if blacklist: |
| 249 | blacklist.Extend([serial], reason='offline') |
| 250 | |
| 251 | def blacklisting_recovery(device): |
| 252 | if _IsBlacklisted(device.adb.GetDeviceSerial(), blacklist): |
| 253 | logging.debug('%s is blacklisted, skipping recovery.', str(device)) |
| 254 | return |
| 255 | |
| 256 | if str(device) in should_reboot_device: |
| 257 | try: |
| 258 | device.WaitUntilFullyBooted(retries=0) |
| 259 | return |
| 260 | except (device_errors.CommandTimeoutError, |
| 261 | device_errors.CommandFailedError): |
| 262 | logging.exception('Failure while waiting for %s. ' |
| 263 | 'Attempting to recover.', str(device)) |
| 264 | |
| 265 | try: |
| 266 | try: |
| 267 | device.Reboot(block=False, timeout=5, retries=0) |
| 268 | except device_errors.CommandTimeoutError: |
| 269 | logging.warning('Timed out while attempting to reboot %s normally.' |
| 270 | 'Attempting alternative reboot.', str(device)) |
| 271 | # The device drops offline before we can grab the exit code, so |
| 272 | # we don't check for status. |
| 273 | device.adb.Root() |
| 274 | device.adb.Shell('echo b > /proc/sysrq-trigger', expect_status=None, |
| 275 | timeout=5, retries=0) |
| 276 | except device_errors.CommandFailedError: |
| 277 | logging.exception('Failed to reboot %s.', str(device)) |
| 278 | if blacklist: |
| 279 | blacklist.Extend([device.adb.GetDeviceSerial()], |
| 280 | reason='reboot_failure') |
| 281 | except device_errors.CommandTimeoutError: |
| 282 | logging.exception('Timed out while rebooting %s.', str(device)) |
| 283 | if blacklist: |
| 284 | blacklist.Extend([device.adb.GetDeviceSerial()], |
| 285 | reason='reboot_timeout') |
| 286 | |
| 287 | try: |
| 288 | device.WaitUntilFullyBooted(retries=0) |
| 289 | except device_errors.CommandFailedError: |
| 290 | logging.exception('Failure while waiting for %s.', str(device)) |
| 291 | if blacklist: |
| 292 | blacklist.Extend([device.adb.GetDeviceSerial()], |
| 293 | reason='reboot_failure') |
| 294 | except device_errors.CommandTimeoutError: |
| 295 | logging.exception('Timed out while waiting for %s.', str(device)) |
| 296 | if blacklist: |
| 297 | blacklist.Extend([device.adb.GetDeviceSerial()], |
| 298 | reason='reboot_timeout') |
| 299 | |
| 300 | device_utils.DeviceUtils.parallel(devices).pMap(blacklisting_recovery) |
| 301 | |
| 302 | |
| 303 | def main(): |
| 304 | parser = argparse.ArgumentParser() |
| 305 | parser.add_argument('--out-dir', |
| 306 | help='Directory where the device path is stored', |
| 307 | default=os.path.join(host_paths.DIR_SOURCE_ROOT, 'out')) |
| 308 | parser.add_argument('--restart-usb', action='store_true', |
| 309 | help='DEPRECATED. ' |
| 310 | 'This script now always tries to reset USB.') |
| 311 | parser.add_argument('--json-output', |
| 312 | help='Output JSON information into a specified file.') |
| 313 | parser.add_argument('--adb-path', |
| 314 | help='Absolute path to the adb binary to use.') |
| 315 | parser.add_argument('--blacklist-file', help='Device blacklist JSON file.') |
| 316 | parser.add_argument('--known-devices-file', action='append', default=[], |
| 317 | dest='known_devices_files', |
| 318 | help='Path to known device lists.') |
| 319 | parser.add_argument('-v', '--verbose', action='count', default=1, |
| 320 | help='Log more information.') |
| 321 | |
| 322 | args = parser.parse_args() |
| 323 | |
| 324 | run_tests_helper.SetLogLevel(args.verbose) |
| 325 | |
| 326 | devil_custom_deps = None |
| 327 | if args.adb_path: |
| 328 | devil_custom_deps = { |
| 329 | 'adb': { |
| 330 | devil_env.GetPlatform(): [args.adb_path], |
| 331 | }, |
| 332 | } |
| 333 | |
| 334 | devil_chromium.Initialize(custom_deps=devil_custom_deps) |
| 335 | |
| 336 | blacklist = (device_blacklist.Blacklist(args.blacklist_file) |
| 337 | if args.blacklist_file |
| 338 | else None) |
| 339 | |
| 340 | last_devices_path = os.path.join( |
| 341 | args.out_dir, device_list.LAST_DEVICES_FILENAME) |
| 342 | args.known_devices_files.append(last_devices_path) |
| 343 | |
| 344 | expected_devices = set() |
| 345 | try: |
| 346 | for path in args.known_devices_files: |
| 347 | if os.path.exists(path): |
| 348 | expected_devices.update(device_list.GetPersistentDeviceList(path)) |
| 349 | except IOError: |
| 350 | logging.warning('Problem reading %s, skipping.', path) |
| 351 | |
| 352 | logging.info('Expected devices:') |
| 353 | for device in expected_devices: |
| 354 | logging.info(' %s', device) |
| 355 | |
| 356 | usb_devices = set(lsusb.get_android_devices()) |
| 357 | devices = [device_utils.DeviceUtils(s) |
| 358 | for s in expected_devices.union(usb_devices)] |
| 359 | |
| 360 | RecoverDevices(devices, blacklist) |
| 361 | statuses = DeviceStatus(devices, blacklist) |
| 362 | |
| 363 | # Log the state of all devices. |
| 364 | for status in statuses: |
| 365 | logging.info(status['serial']) |
| 366 | adb_status = status.get('adb_status') |
| 367 | blacklisted = status.get('blacklisted') |
| 368 | logging.info(' USB status: %s', |
| 369 | 'online' if status.get('usb_status') else 'offline') |
| 370 | logging.info(' ADB status: %s', adb_status) |
| 371 | logging.info(' Blacklisted: %s', str(blacklisted)) |
| 372 | if adb_status == 'device' and not blacklisted: |
| 373 | logging.info(' Device type: %s', status.get('ro.build.product')) |
| 374 | logging.info(' OS build: %s', status.get('ro.build.id')) |
| 375 | logging.info(' OS build fingerprint: %s', |
| 376 | status.get('ro.build.fingerprint')) |
| 377 | logging.info(' Battery state:') |
| 378 | for k, v in status.get('battery', {}).iteritems(): |
| 379 | logging.info(' %s: %s', k, v) |
| 380 | logging.info(' IMEI slice: %s', status.get('imei_slice')) |
| 381 | logging.info(' WiFi IP: %s', status.get('wifi_ip')) |
| 382 | |
| 383 | # Update the last devices file(s). |
| 384 | for path in args.known_devices_files: |
| 385 | device_list.WritePersistentDeviceList( |
| 386 | path, [status['serial'] for status in statuses]) |
| 387 | |
| 388 | # Write device info to file for buildbot info display. |
| 389 | if os.path.exists('/home/chrome-bot'): |
| 390 | with open('/home/chrome-bot/.adb_device_info', 'w') as f: |
| 391 | for status in statuses: |
| 392 | try: |
| 393 | if status['adb_status'] == 'device': |
| 394 | f.write('{serial} {adb_status} {build_product} {build_id} ' |
| 395 | '{temperature:.1f}C {level}%\n'.format( |
| 396 | serial=status['serial'], |
| 397 | adb_status=status['adb_status'], |
| 398 | build_product=status['type'], |
| 399 | build_id=status['build'], |
| 400 | temperature=float(status['battery']['temperature']) / 10, |
| 401 | level=status['battery']['level'] |
| 402 | )) |
| 403 | elif status.get('usb_status', False): |
| 404 | f.write('{serial} {adb_status}\n'.format( |
| 405 | serial=status['serial'], |
| 406 | adb_status=status['adb_status'] |
| 407 | )) |
| 408 | else: |
| 409 | f.write('{serial} offline\n'.format( |
| 410 | serial=status['serial'] |
| 411 | )) |
| 412 | except Exception: # pylint: disable=broad-except |
| 413 | pass |
| 414 | |
| 415 | # Dump the device statuses to JSON. |
| 416 | if args.json_output: |
| 417 | with open(args.json_output, 'wb') as f: |
| 418 | f.write(json.dumps(statuses, indent=4)) |
| 419 | |
| 420 | live_devices = [status['serial'] for status in statuses |
| 421 | if (status['adb_status'] == 'device' |
| 422 | and not _IsBlacklisted(status['serial'], blacklist))] |
| 423 | |
| 424 | # If all devices failed, or if there are no devices, it's an infra error. |
| 425 | return 0 if live_devices else exit_codes.INFRA |
| 426 | |
| 427 | |
| 428 | if __name__ == '__main__': |
| 429 | sys.exit(main()) |