Ben Murdoch | 7dbb3d5 | 2013-07-17 14:55:54 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # |
| 3 | # Copyright 2013 The Chromium Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | |
| 7 | """A class to keep track of devices across builds and report state.""" |
| 8 | import logging |
| 9 | import optparse |
| 10 | import os |
| 11 | import smtplib |
| 12 | import sys |
| 13 | import re |
| 14 | |
| 15 | import bb_annotations |
| 16 | |
| 17 | sys.path.append(os.path.join(os.path.dirname(__file__), '..')) |
| 18 | from pylib import android_commands |
| 19 | from pylib import constants |
Ben Murdoch | a3f7b4e | 2013-07-24 10:36:34 +0100 | [diff] [blame] | 20 | from pylib import perf_tests_helper |
Ben Murdoch | 7dbb3d5 | 2013-07-17 14:55:54 +0100 | [diff] [blame] | 21 | from pylib.cmd_helper import GetCmdOutput |
| 22 | |
| 23 | |
| 24 | def DeviceInfo(serial, options): |
| 25 | """Gathers info on a device via various adb calls. |
| 26 | |
| 27 | Args: |
| 28 | serial: The serial of the attached device to construct info about. |
| 29 | |
| 30 | Returns: |
| 31 | Tuple of device type, build id, report as a string, error messages, and |
| 32 | boolean indicating whether or not device can be used for testing. |
| 33 | """ |
| 34 | |
Ben Murdoch | 7dbb3d5 | 2013-07-17 14:55:54 +0100 | [diff] [blame] | 35 | device_adb = android_commands.AndroidCommands(serial) |
| 36 | |
| 37 | # TODO(navabi): Replace AdbShellCmd with device_adb. |
Ben Murdoch | a3f7b4e | 2013-07-24 10:36:34 +0100 | [diff] [blame] | 38 | device_type = device_adb.GetBuildProduct() |
| 39 | device_build = device_adb.GetBuildId() |
| 40 | device_build_type = device_adb.GetBuildType() |
| 41 | device_product_name = device_adb.GetProductName() |
Ben Murdoch | 7dbb3d5 | 2013-07-17 14:55:54 +0100 | [diff] [blame] | 42 | |
Ben Murdoch | a3f7b4e | 2013-07-24 10:36:34 +0100 | [diff] [blame] | 43 | setup_wizard_disabled = device_adb.GetSetupWizardStatus() == 'DISABLED' |
| 44 | battery = device_adb.GetBatteryInfo() |
Ben Murdoch | 7dbb3d5 | 2013-07-17 14:55:54 +0100 | [diff] [blame] | 45 | install_output = GetCmdOutput( |
| 46 | ['%s/build/android/adb_install_apk.py' % constants.DIR_SOURCE_ROOT, '--apk', |
| 47 | '%s/build/android/CheckInstallApk-debug.apk' % constants.DIR_SOURCE_ROOT]) |
Ben Murdoch | 58e6fbe | 2013-07-26 10:20:38 +0100 | [diff] [blame^] | 48 | |
| 49 | def _GetData(re_expression, line, lambda_function=lambda x:x): |
| 50 | if not line: |
| 51 | return 'Unknown' |
| 52 | found = re.findall(re_expression, line) |
| 53 | if found and len(found): |
| 54 | return lambda_function(found[0]) |
| 55 | return 'Unknown' |
| 56 | |
| 57 | install_speed = _GetData('(\d+) KB/s', install_output) |
| 58 | ac_power = _GetData('AC powered: (\w+)', battery) |
| 59 | battery_level = _GetData('level: (\d+)', battery) |
| 60 | battery_temp = _GetData('temperature: (\d+)', battery, |
| 61 | lambda x: float(x) / 10.0) |
| 62 | imei_slice = _GetData('Device ID = (\d+)', |
| 63 | device_adb.GetSubscriberInfo(), |
| 64 | lambda x: x[-6:]) |
Ben Murdoch | 7dbb3d5 | 2013-07-17 14:55:54 +0100 | [diff] [blame] | 65 | report = ['Device %s (%s)' % (serial, device_type), |
Ben Murdoch | a3f7b4e | 2013-07-24 10:36:34 +0100 | [diff] [blame] | 66 | ' Build: %s (%s)' % |
| 67 | (device_build, device_adb.GetBuildFingerprint()), |
Ben Murdoch | 7dbb3d5 | 2013-07-17 14:55:54 +0100 | [diff] [blame] | 68 | ' Battery: %s%%' % battery_level, |
| 69 | ' Battery temp: %s' % battery_temp, |
Ben Murdoch | a3f7b4e | 2013-07-24 10:36:34 +0100 | [diff] [blame] | 70 | ' IMEI slice: %s' % imei_slice, |
| 71 | ' Wifi IP: %s' % device_adb.GetWifiIP(), |
Ben Murdoch | 7dbb3d5 | 2013-07-17 14:55:54 +0100 | [diff] [blame] | 72 | ' Install Speed: %s KB/s' % install_speed, |
| 73 | ''] |
| 74 | |
| 75 | errors = [] |
| 76 | if battery_level < 15: |
| 77 | errors += ['Device critically low in battery. Turning off device.'] |
| 78 | if (not setup_wizard_disabled and device_build_type != 'user' and |
| 79 | not options.no_provisioning_check): |
| 80 | errors += ['Setup wizard not disabled. Was it provisioned correctly?'] |
| 81 | if device_product_name == 'mantaray' and ac_power != 'true': |
| 82 | errors += ['Mantaray device not connected to AC power.'] |
| 83 | # TODO(navabi): Insert warning once we have a better handle of what install |
| 84 | # speeds to expect. The following lines were causing too many alerts. |
| 85 | # if install_speed < 500: |
| 86 | # errors += ['Device install speed too low. Do not use for testing.'] |
| 87 | |
| 88 | # Causing the device status check step fail for slow install speed or low |
| 89 | # battery currently is too disruptive to the bots (especially try bots). |
| 90 | # Turn off devices with low battery and the step does not fail. |
| 91 | if battery_level < 15: |
| 92 | device_adb.EnableAdbRoot() |
Ben Murdoch | a3f7b4e | 2013-07-24 10:36:34 +0100 | [diff] [blame] | 93 | device_adb.Shutdown() |
| 94 | full_report = '\n'.join(report) |
| 95 | return device_type, device_build, battery_level, full_report, errors, True |
Ben Murdoch | 7dbb3d5 | 2013-07-17 14:55:54 +0100 | [diff] [blame] | 96 | |
| 97 | |
| 98 | def CheckForMissingDevices(options, adb_online_devs): |
| 99 | """Uses file of previous online devices to detect broken phones. |
| 100 | |
| 101 | Args: |
| 102 | options: out_dir parameter of options argument is used as the base |
| 103 | directory to load and update the cache file. |
| 104 | adb_online_devs: A list of serial numbers of the currently visible |
| 105 | and online attached devices. |
| 106 | """ |
| 107 | # TODO(navabi): remove this once the bug that causes different number |
| 108 | # of devices to be detected between calls is fixed. |
| 109 | logger = logging.getLogger() |
| 110 | logger.setLevel(logging.INFO) |
| 111 | |
| 112 | out_dir = os.path.abspath(options.out_dir) |
| 113 | |
| 114 | def ReadDeviceList(file_name): |
| 115 | devices_path = os.path.join(out_dir, file_name) |
| 116 | devices = [] |
| 117 | try: |
| 118 | with open(devices_path) as f: |
| 119 | devices = f.read().splitlines() |
| 120 | except IOError: |
| 121 | # Ignore error, file might not exist |
| 122 | pass |
| 123 | return devices |
| 124 | |
| 125 | def WriteDeviceList(file_name, device_list): |
| 126 | path = os.path.join(out_dir, file_name) |
| 127 | if not os.path.exists(out_dir): |
| 128 | os.makedirs(out_dir) |
| 129 | with open(path, 'w') as f: |
| 130 | # Write devices currently visible plus devices previously seen. |
| 131 | f.write('\n'.join(set(device_list))) |
| 132 | |
| 133 | last_devices_path = os.path.join(out_dir, '.last_devices') |
| 134 | last_devices = ReadDeviceList('.last_devices') |
| 135 | missing_devs = list(set(last_devices) - set(adb_online_devs)) |
| 136 | |
| 137 | all_known_devices = list(set(adb_online_devs) | set(last_devices)) |
| 138 | WriteDeviceList('.last_devices', all_known_devices) |
| 139 | WriteDeviceList('.last_missing', missing_devs) |
| 140 | |
| 141 | if not all_known_devices: |
| 142 | # This can happen if for some reason the .last_devices file is not |
| 143 | # present or if it was empty. |
| 144 | return ['No online devices. Have any devices been plugged in?'] |
| 145 | if missing_devs: |
| 146 | devices_missing_msg = '%d devices not detected.' % len(missing_devs) |
| 147 | bb_annotations.PrintSummaryText(devices_missing_msg) |
| 148 | |
| 149 | # TODO(navabi): Debug by printing both output from GetCmdOutput and |
| 150 | # GetAttachedDevices to compare results. |
| 151 | return ['Current online devices: %s' % adb_online_devs, |
| 152 | '%s are no longer visible. Were they removed?\n' % missing_devs, |
| 153 | 'SHERIFF: See go/chrome_device_monitor', |
| 154 | 'Cache file: %s\n\n' % last_devices_path, |
| 155 | 'adb devices: %s' % GetCmdOutput(['adb', 'devices']), |
| 156 | 'adb devices(GetAttachedDevices): %s' % |
| 157 | android_commands.GetAttachedDevices()] |
| 158 | else: |
| 159 | new_devs = set(adb_online_devs) - set(last_devices) |
| 160 | if new_devs and os.path.exists(last_devices_path): |
| 161 | bb_annotations.PrintWarning() |
| 162 | bb_annotations.PrintSummaryText( |
| 163 | '%d new devices detected' % len(new_devs)) |
| 164 | print ('New devices detected %s. And now back to your ' |
| 165 | 'regularly scheduled program.' % list(new_devs)) |
| 166 | |
| 167 | |
| 168 | def SendDeviceStatusAlert(msg): |
| 169 | from_address = 'buildbot@chromium.org' |
| 170 | to_address = 'chromium-android-device-alerts@google.com' |
| 171 | bot_name = os.environ.get('BUILDBOT_BUILDERNAME') |
| 172 | slave_name = os.environ.get('BUILDBOT_SLAVENAME') |
| 173 | subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name) |
| 174 | msg_body = '\r\n'.join(['From: %s' % from_address, 'To: %s' % to_address, |
| 175 | 'Subject: %s' % subject, '', msg]) |
| 176 | try: |
| 177 | server = smtplib.SMTP('localhost') |
| 178 | server.sendmail(from_address, [to_address], msg_body) |
| 179 | server.quit() |
| 180 | except Exception as e: |
| 181 | print 'Failed to send alert email. Error: %s' % e |
| 182 | |
| 183 | |
| 184 | def main(): |
| 185 | parser = optparse.OptionParser() |
| 186 | parser.add_option('', '--out-dir', |
| 187 | help='Directory where the device path is stored', |
Ben Murdoch | ca12bfa | 2013-07-23 11:17:05 +0100 | [diff] [blame] | 188 | default=os.path.join(constants.DIR_SOURCE_ROOT, 'out')) |
Ben Murdoch | 7dbb3d5 | 2013-07-17 14:55:54 +0100 | [diff] [blame] | 189 | parser.add_option('--no-provisioning-check', |
| 190 | help='Will not check if devices are provisioned properly.') |
Ben Murdoch | a3f7b4e | 2013-07-24 10:36:34 +0100 | [diff] [blame] | 191 | parser.add_option('--device-status-dashboard', |
| 192 | help='Output device status data for dashboard.') |
Ben Murdoch | 7dbb3d5 | 2013-07-17 14:55:54 +0100 | [diff] [blame] | 193 | options, args = parser.parse_args() |
| 194 | if args: |
| 195 | parser.error('Unknown options %s' % args) |
| 196 | devices = android_commands.GetAttachedDevices() |
Ben Murdoch | a3f7b4e | 2013-07-24 10:36:34 +0100 | [diff] [blame] | 197 | # TODO(navabi): Test to make sure this fails and then fix call |
| 198 | offline_devices = android_commands.GetAttachedDevices(hardware=False, |
| 199 | emulator=False, |
| 200 | offline=True) |
| 201 | |
| 202 | types, builds, batteries, reports, errors = [], [], [], [], [] |
Ben Murdoch | 7dbb3d5 | 2013-07-17 14:55:54 +0100 | [diff] [blame] | 203 | fail_step_lst = [] |
| 204 | if devices: |
Ben Murdoch | a3f7b4e | 2013-07-24 10:36:34 +0100 | [diff] [blame] | 205 | types, builds, batteries, reports, errors, fail_step_lst = ( |
Ben Murdoch | 7dbb3d5 | 2013-07-17 14:55:54 +0100 | [diff] [blame] | 206 | zip(*[DeviceInfo(dev, options) for dev in devices])) |
| 207 | |
| 208 | err_msg = CheckForMissingDevices(options, devices) or [] |
| 209 | |
| 210 | unique_types = list(set(types)) |
| 211 | unique_builds = list(set(builds)) |
| 212 | |
| 213 | bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' |
| 214 | % (len(devices), unique_types, unique_builds)) |
| 215 | print '\n'.join(reports) |
| 216 | |
| 217 | for serial, dev_errors in zip(devices, errors): |
| 218 | if dev_errors: |
| 219 | err_msg += ['%s errors:' % serial] |
| 220 | err_msg += [' %s' % error for error in dev_errors] |
| 221 | |
| 222 | if err_msg: |
| 223 | bb_annotations.PrintWarning() |
| 224 | msg = '\n'.join(err_msg) |
| 225 | print msg |
| 226 | SendDeviceStatusAlert(msg) |
| 227 | |
Ben Murdoch | a3f7b4e | 2013-07-24 10:36:34 +0100 | [diff] [blame] | 228 | if options.device_status_dashboard: |
| 229 | perf_tests_helper.PrintPerfResult('BotDevices', 'OnlineDevices', |
| 230 | [len(devices)], 'devices') |
| 231 | perf_tests_helper.PrintPerfResult('BotDevices', 'OfflineDevices', |
| 232 | [len(offline_devices)], 'devices', |
| 233 | 'unimportant') |
| 234 | for serial, battery in zip(devices, batteries): |
| 235 | perf_tests_helper.PrintPerfResult('DeviceBattery', serial, [battery], '%', |
| 236 | 'unimportant') |
| 237 | |
Ben Murdoch | 7dbb3d5 | 2013-07-17 14:55:54 +0100 | [diff] [blame] | 238 | if False in fail_step_lst: |
| 239 | # TODO(navabi): Build fails on device status check step if there exists any |
| 240 | # devices with critically low battery or install speed. Remove those devices |
| 241 | # from testing, allowing build to continue with good devices. |
| 242 | return 1 |
| 243 | |
| 244 | if not devices: |
| 245 | return 1 |
| 246 | |
| 247 | |
| 248 | if __name__ == '__main__': |
| 249 | sys.exit(main()) |