Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame^] | 1 | # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | """Run specific test on specific environment.""" |
| 6 | |
| 7 | import json |
| 8 | import logging |
| 9 | import os |
| 10 | import re |
| 11 | import shutil |
| 12 | import string |
| 13 | import tempfile |
| 14 | import time |
| 15 | import zipfile |
| 16 | |
| 17 | from devil.utils import zip_utils |
| 18 | from pylib.base import base_test_result |
| 19 | from pylib.base import test_run |
| 20 | from pylib.remote.device import appurify_constants |
| 21 | from pylib.remote.device import appurify_sanitized |
| 22 | from pylib.remote.device import remote_device_helper |
| 23 | |
| 24 | _DEVICE_OFFLINE_RE = re.compile('error: device not found') |
| 25 | _LONG_MSG_RE = re.compile('longMsg=(.*)$') |
| 26 | _SHORT_MSG_RE = re.compile('shortMsg=(.*)$') |
| 27 | |
| 28 | class RemoteDeviceTestRun(test_run.TestRun): |
| 29 | """Run tests on a remote device.""" |
| 30 | |
| 31 | _TEST_RUN_KEY = 'test_run' |
| 32 | _TEST_RUN_ID_KEY = 'test_run_id' |
| 33 | |
| 34 | WAIT_TIME = 5 |
| 35 | COMPLETE = 'complete' |
| 36 | HEARTBEAT_INTERVAL = 300 |
| 37 | |
| 38 | def __init__(self, env, test_instance): |
| 39 | """Constructor. |
| 40 | |
| 41 | Args: |
| 42 | env: Environment the tests will run in. |
| 43 | test_instance: The test that will be run. |
| 44 | """ |
| 45 | super(RemoteDeviceTestRun, self).__init__(env, test_instance) |
| 46 | self._env = env |
| 47 | self._test_instance = test_instance |
| 48 | self._app_id = '' |
| 49 | self._test_id = '' |
| 50 | self._results = '' |
| 51 | self._test_run_id = '' |
| 52 | self._results_temp_dir = None |
| 53 | |
| 54 | #override |
| 55 | def SetUp(self): |
| 56 | """Set up a test run.""" |
| 57 | if self._env.trigger: |
| 58 | self._TriggerSetUp() |
| 59 | elif self._env.collect: |
| 60 | assert isinstance(self._env.collect, basestring), ( |
| 61 | 'File for storing test_run_id must be a string.') |
| 62 | with open(self._env.collect, 'r') as persisted_data_file: |
| 63 | persisted_data = json.loads(persisted_data_file.read()) |
| 64 | self._env.LoadFrom(persisted_data) |
| 65 | self.LoadFrom(persisted_data) |
| 66 | |
| 67 | def _TriggerSetUp(self): |
| 68 | """Set up the triggering of a test run.""" |
| 69 | raise NotImplementedError |
| 70 | |
| 71 | #override |
| 72 | def RunTests(self): |
| 73 | """Run the test.""" |
| 74 | if self._env.trigger: |
| 75 | with appurify_sanitized.SanitizeLogging(self._env.verbose_count, |
| 76 | logging.WARNING): |
| 77 | test_start_res = appurify_sanitized.api.tests_run( |
| 78 | self._env.token, self._env.device_type_id, self._app_id, |
| 79 | self._test_id) |
| 80 | remote_device_helper.TestHttpResponse( |
| 81 | test_start_res, 'Unable to run test.') |
| 82 | self._test_run_id = test_start_res.json()['response']['test_run_id'] |
| 83 | logging.info('Test run id: %s', self._test_run_id) |
| 84 | |
| 85 | if self._env.collect: |
| 86 | current_status = '' |
| 87 | timeout_counter = 0 |
| 88 | heartbeat_counter = 0 |
| 89 | while self._GetTestStatus(self._test_run_id) != self.COMPLETE: |
| 90 | if self._results['detailed_status'] != current_status: |
| 91 | logging.info('Test status: %s', self._results['detailed_status']) |
| 92 | current_status = self._results['detailed_status'] |
| 93 | timeout_counter = 0 |
| 94 | heartbeat_counter = 0 |
| 95 | if heartbeat_counter > self.HEARTBEAT_INTERVAL: |
| 96 | logging.info('Test status: %s', self._results['detailed_status']) |
| 97 | heartbeat_counter = 0 |
| 98 | |
| 99 | timeout = self._env.timeouts.get( |
| 100 | current_status, self._env.timeouts['unknown']) |
| 101 | if timeout_counter > timeout: |
| 102 | raise remote_device_helper.RemoteDeviceError( |
| 103 | 'Timeout while in %s state for %s seconds' |
| 104 | % (current_status, timeout), |
| 105 | is_infra_error=True) |
| 106 | time.sleep(self.WAIT_TIME) |
| 107 | timeout_counter += self.WAIT_TIME |
| 108 | heartbeat_counter += self.WAIT_TIME |
| 109 | self._DownloadTestResults(self._env.results_path) |
| 110 | |
| 111 | if self._results['results']['exception']: |
| 112 | raise remote_device_helper.RemoteDeviceError( |
| 113 | self._results['results']['exception'], is_infra_error=True) |
| 114 | |
| 115 | return self._ParseTestResults() |
| 116 | |
| 117 | #override |
| 118 | def TearDown(self): |
| 119 | """Tear down the test run.""" |
| 120 | if self._env.collect: |
| 121 | self._CollectTearDown() |
| 122 | elif self._env.trigger: |
| 123 | assert isinstance(self._env.trigger, basestring), ( |
| 124 | 'File for storing test_run_id must be a string.') |
| 125 | with open(self._env.trigger, 'w') as persisted_data_file: |
| 126 | persisted_data = {} |
| 127 | self.DumpTo(persisted_data) |
| 128 | self._env.DumpTo(persisted_data) |
| 129 | persisted_data_file.write(json.dumps(persisted_data)) |
| 130 | |
| 131 | def _CollectTearDown(self): |
| 132 | if self._GetTestStatus(self._test_run_id) != self.COMPLETE: |
| 133 | with appurify_sanitized.SanitizeLogging(self._env.verbose_count, |
| 134 | logging.WARNING): |
| 135 | test_abort_res = appurify_sanitized.api.tests_abort( |
| 136 | self._env.token, self._test_run_id, reason='Test runner exiting.') |
| 137 | remote_device_helper.TestHttpResponse(test_abort_res, |
| 138 | 'Unable to abort test.') |
| 139 | if self._results_temp_dir: |
| 140 | shutil.rmtree(self._results_temp_dir) |
| 141 | |
| 142 | def __enter__(self): |
| 143 | """Set up the test run when used as a context manager.""" |
| 144 | self.SetUp() |
| 145 | return self |
| 146 | |
| 147 | def __exit__(self, exc_type, exc_val, exc_tb): |
| 148 | """Tear down the test run when used as a context manager.""" |
| 149 | self.TearDown() |
| 150 | |
| 151 | def DumpTo(self, persisted_data): |
| 152 | test_run_data = { |
| 153 | self._TEST_RUN_ID_KEY: self._test_run_id, |
| 154 | } |
| 155 | persisted_data[self._TEST_RUN_KEY] = test_run_data |
| 156 | |
| 157 | def LoadFrom(self, persisted_data): |
| 158 | test_run_data = persisted_data[self._TEST_RUN_KEY] |
| 159 | self._test_run_id = test_run_data[self._TEST_RUN_ID_KEY] |
| 160 | |
| 161 | def _ParseTestResults(self): |
| 162 | raise NotImplementedError |
| 163 | |
| 164 | def _GetTestByName(self, test_name): |
| 165 | """Gets test_id for specific test. |
| 166 | |
| 167 | Args: |
| 168 | test_name: Test to find the ID of. |
| 169 | """ |
| 170 | with appurify_sanitized.SanitizeLogging(self._env.verbose_count, |
| 171 | logging.WARNING): |
| 172 | test_list_res = appurify_sanitized.api.tests_list(self._env.token) |
| 173 | remote_device_helper.TestHttpResponse(test_list_res, |
| 174 | 'Unable to get tests list.') |
| 175 | for test in test_list_res.json()['response']: |
| 176 | if test['test_type'] == test_name: |
| 177 | return test['test_id'] |
| 178 | raise remote_device_helper.RemoteDeviceError( |
| 179 | 'No test found with name %s' % (test_name)) |
| 180 | |
| 181 | def _DownloadTestResults(self, results_path): |
| 182 | """Download the test results from remote device service. |
| 183 | |
| 184 | Downloads results in temporary location, and then copys results |
| 185 | to results_path if results_path is not set to None. |
| 186 | |
| 187 | Args: |
| 188 | results_path: Path to download appurify results zipfile. |
| 189 | |
| 190 | Returns: |
| 191 | Path to downloaded file. |
| 192 | """ |
| 193 | |
| 194 | if self._results_temp_dir is None: |
| 195 | self._results_temp_dir = tempfile.mkdtemp() |
| 196 | logging.info('Downloading results to %s.', self._results_temp_dir) |
| 197 | with appurify_sanitized.SanitizeLogging(self._env.verbose_count, |
| 198 | logging.WARNING): |
| 199 | appurify_sanitized.utils.wget(self._results['results']['url'], |
| 200 | self._results_temp_dir + '/results') |
| 201 | if results_path: |
| 202 | logging.info('Copying results to %s', results_path) |
| 203 | if not os.path.exists(os.path.dirname(results_path)): |
| 204 | os.makedirs(os.path.dirname(results_path)) |
| 205 | shutil.copy(self._results_temp_dir + '/results', results_path) |
| 206 | return self._results_temp_dir + '/results' |
| 207 | |
| 208 | def _GetTestStatus(self, test_run_id): |
| 209 | """Checks the state of the test, and sets self._results |
| 210 | |
| 211 | Args: |
| 212 | test_run_id: Id of test on on remote service. |
| 213 | """ |
| 214 | |
| 215 | with appurify_sanitized.SanitizeLogging(self._env.verbose_count, |
| 216 | logging.WARNING): |
| 217 | test_check_res = appurify_sanitized.api.tests_check_result( |
| 218 | self._env.token, test_run_id) |
| 219 | remote_device_helper.TestHttpResponse(test_check_res, |
| 220 | 'Unable to get test status.') |
| 221 | self._results = test_check_res.json()['response'] |
| 222 | return self._results['status'] |
| 223 | |
| 224 | def _AmInstrumentTestSetup(self, app_path, test_path, runner_package, |
| 225 | environment_variables, extra_apks=None): |
| 226 | config = {'runner': runner_package} |
| 227 | if environment_variables: |
| 228 | config['environment_vars'] = ','.join( |
| 229 | '%s=%s' % (k, v) for k, v in environment_variables.iteritems()) |
| 230 | |
| 231 | self._app_id = self._UploadAppToDevice(app_path) |
| 232 | |
| 233 | data_deps = self._test_instance.GetDataDependencies() |
| 234 | if data_deps: |
| 235 | with tempfile.NamedTemporaryFile(suffix='.zip') as test_with_deps: |
| 236 | sdcard_files = [] |
| 237 | additional_apks = [] |
| 238 | host_test = os.path.basename(test_path) |
| 239 | with zipfile.ZipFile(test_with_deps.name, 'w') as zip_file: |
| 240 | zip_file.write(test_path, host_test, zipfile.ZIP_DEFLATED) |
| 241 | for h, _ in data_deps: |
| 242 | if os.path.isdir(h): |
| 243 | zip_utils.WriteToZipFile(zip_file, h, '.') |
| 244 | sdcard_files.extend(os.listdir(h)) |
| 245 | else: |
| 246 | zip_utils.WriteToZipFile(zip_file, h, os.path.basename(h)) |
| 247 | sdcard_files.append(os.path.basename(h)) |
| 248 | for a in extra_apks or (): |
| 249 | zip_utils.WriteToZipFile(zip_file, a, os.path.basename(a)) |
| 250 | additional_apks.append(os.path.basename(a)) |
| 251 | |
| 252 | config['sdcard_files'] = ','.join(sdcard_files) |
| 253 | config['host_test'] = host_test |
| 254 | if additional_apks: |
| 255 | config['additional_apks'] = ','.join(additional_apks) |
| 256 | self._test_id = self._UploadTestToDevice( |
| 257 | 'robotium', test_with_deps.name, app_id=self._app_id) |
| 258 | else: |
| 259 | self._test_id = self._UploadTestToDevice('robotium', test_path) |
| 260 | |
| 261 | logging.info('Setting config: %s', config) |
| 262 | appurify_configs = {} |
| 263 | if self._env.network_config: |
| 264 | appurify_configs['network'] = self._env.network_config |
| 265 | self._SetTestConfig('robotium', config, **appurify_configs) |
| 266 | |
| 267 | def _UploadAppToDevice(self, app_path): |
| 268 | """Upload app to device.""" |
| 269 | logging.info('Uploading %s to remote service as %s.', app_path, |
| 270 | self._test_instance.suite) |
| 271 | with open(app_path, 'rb') as apk_src: |
| 272 | with appurify_sanitized.SanitizeLogging(self._env.verbose_count, |
| 273 | logging.WARNING): |
| 274 | upload_results = appurify_sanitized.api.apps_upload( |
| 275 | self._env.token, apk_src, 'raw', name=self._test_instance.suite) |
| 276 | remote_device_helper.TestHttpResponse( |
| 277 | upload_results, 'Unable to upload %s.' % app_path) |
| 278 | return upload_results.json()['response']['app_id'] |
| 279 | |
| 280 | def _UploadTestToDevice(self, test_type, test_path, app_id=None): |
| 281 | """Upload test to device |
| 282 | Args: |
| 283 | test_type: Type of test that is being uploaded. Ex. uirobot, gtest.. |
| 284 | """ |
| 285 | logging.info('Uploading %s to remote service.', test_path) |
| 286 | with open(test_path, 'rb') as test_src: |
| 287 | with appurify_sanitized.SanitizeLogging(self._env.verbose_count, |
| 288 | logging.WARNING): |
| 289 | upload_results = appurify_sanitized.api.tests_upload( |
| 290 | self._env.token, test_src, 'raw', test_type, app_id=app_id) |
| 291 | remote_device_helper.TestHttpResponse(upload_results, |
| 292 | 'Unable to upload %s.' % test_path) |
| 293 | return upload_results.json()['response']['test_id'] |
| 294 | |
| 295 | def _SetTestConfig(self, runner_type, runner_configs, |
| 296 | network=appurify_constants.NETWORK.WIFI_1_BAR, |
| 297 | pcap=0, profiler=0, videocapture=0): |
| 298 | """Generates and uploads config file for test. |
| 299 | Args: |
| 300 | runner_configs: Configs specific to the runner you are using. |
| 301 | network: Config to specify the network environment the devices running |
| 302 | the tests will be in. |
| 303 | pcap: Option to set the recording the of network traffic from the device. |
| 304 | profiler: Option to set the recording of CPU, memory, and network |
| 305 | transfer usage in the tests. |
| 306 | videocapture: Option to set video capture during the tests. |
| 307 | |
| 308 | """ |
| 309 | logging.info('Generating config file for test.') |
| 310 | with tempfile.TemporaryFile() as config: |
| 311 | config_data = [ |
| 312 | '[appurify]', |
| 313 | 'network=%s' % network, |
| 314 | 'pcap=%s' % pcap, |
| 315 | 'profiler=%s' % profiler, |
| 316 | 'videocapture=%s' % videocapture, |
| 317 | '[%s]' % runner_type |
| 318 | ] |
| 319 | config_data.extend( |
| 320 | '%s=%s' % (k, v) for k, v in runner_configs.iteritems()) |
| 321 | config.write(''.join('%s\n' % l for l in config_data)) |
| 322 | config.flush() |
| 323 | config.seek(0) |
| 324 | with appurify_sanitized.SanitizeLogging(self._env.verbose_count, |
| 325 | logging.WARNING): |
| 326 | config_response = appurify_sanitized.api.config_upload( |
| 327 | self._env.token, config, self._test_id) |
| 328 | remote_device_helper.TestHttpResponse( |
| 329 | config_response, 'Unable to upload test config.') |
| 330 | |
| 331 | def _LogLogcat(self, level=logging.CRITICAL): |
| 332 | """Prints out logcat downloaded from remote service. |
| 333 | Args: |
| 334 | level: logging level to print at. |
| 335 | |
| 336 | Raises: |
| 337 | KeyError: If appurify_results/logcat.txt file cannot be found in |
| 338 | downloaded zip. |
| 339 | """ |
| 340 | zip_file = self._DownloadTestResults(None) |
| 341 | with zipfile.ZipFile(zip_file) as z: |
| 342 | try: |
| 343 | logcat = z.read('appurify_results/logcat.txt') |
| 344 | printable_logcat = ''.join(c for c in logcat if c in string.printable) |
| 345 | for line in printable_logcat.splitlines(): |
| 346 | logging.log(level, line) |
| 347 | except KeyError: |
| 348 | logging.error('No logcat found.') |
| 349 | |
| 350 | def _LogAdbTraceLog(self): |
| 351 | zip_file = self._DownloadTestResults(None) |
| 352 | with zipfile.ZipFile(zip_file) as z: |
| 353 | adb_trace_log = z.read('adb_trace.log') |
| 354 | for line in adb_trace_log.splitlines(): |
| 355 | logging.critical(line) |
| 356 | |
| 357 | def _DidDeviceGoOffline(self): |
| 358 | zip_file = self._DownloadTestResults(None) |
| 359 | with zipfile.ZipFile(zip_file) as z: |
| 360 | adb_trace_log = z.read('adb_trace.log') |
| 361 | if any(_DEVICE_OFFLINE_RE.search(l) for l in adb_trace_log.splitlines()): |
| 362 | return True |
| 363 | return False |
| 364 | |
| 365 | def _DetectPlatformErrors(self, results): |
| 366 | if not self._results['results']['pass']: |
| 367 | crash_msg = None |
| 368 | for line in self._results['results']['output'].splitlines(): |
| 369 | m = _LONG_MSG_RE.search(line) |
| 370 | if m: |
| 371 | crash_msg = m.group(1) |
| 372 | break |
| 373 | m = _SHORT_MSG_RE.search(line) |
| 374 | if m: |
| 375 | crash_msg = m.group(1) |
| 376 | if crash_msg: |
| 377 | self._LogLogcat() |
| 378 | results.AddResult(base_test_result.BaseTestResult( |
| 379 | crash_msg, base_test_result.ResultType.CRASH)) |
| 380 | elif self._DidDeviceGoOffline(): |
| 381 | self._LogLogcat() |
| 382 | self._LogAdbTraceLog() |
| 383 | raise remote_device_helper.RemoteDeviceError( |
| 384 | 'Remote service unable to reach device.', is_infra_error=True) |
| 385 | else: |
| 386 | # Remote service is reporting a failure, but no failure in results obj. |
| 387 | if results.DidRunPass(): |
| 388 | results.AddResult(base_test_result.BaseTestResult( |
| 389 | 'Remote service detected error.', |
| 390 | base_test_result.ResultType.UNKNOWN)) |