Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 1 | #!/usr/bin/python2.4 |
| 2 | # |
| 3 | # |
| 4 | # Copyright 2008, The Android Open Source Project |
| 5 | # |
| 6 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 7 | # you may not use this file except in compliance with the License. |
| 8 | # You may obtain a copy of the License at |
| 9 | # |
| 10 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | # |
| 12 | # Unless required by applicable law or agreed to in writing, software |
| 13 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | # See the License for the specific language governing permissions and |
| 16 | # limitations under the License. |
| 17 | |
| 18 | """TestSuite definition for Android instrumentation tests.""" |
| 19 | |
| 20 | # python imports |
| 21 | import os |
| 22 | |
| 23 | # local imports |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 24 | import coverage |
| 25 | import errors |
| 26 | import logger |
Brett Chabot | 924c089 | 2009-10-21 14:23:54 -0700 | [diff] [blame] | 27 | import test_suite |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 28 | |
| 29 | |
Brett Chabot | 924c089 | 2009-10-21 14:23:54 -0700 | [diff] [blame] | 30 | class InstrumentationTestSuite(test_suite.AbstractTestSuite): |
| 31 | """Represents a java instrumentation test suite definition run on device.""" |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 32 | |
Brett Chabot | 924c089 | 2009-10-21 14:23:54 -0700 | [diff] [blame] | 33 | DEFAULT_RUNNER = "android.test.InstrumentationTestRunner" |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 34 | |
Guang Zhu | 863870c | 2010-04-15 10:48:26 -0700 | [diff] [blame] | 35 | # dependency on libcore (used for Emma) |
| 36 | _LIBCORE_BUILD_PATH = "libcore" |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 37 | |
Brett Chabot | 924c089 | 2009-10-21 14:23:54 -0700 | [diff] [blame] | 38 | def __init__(self): |
| 39 | test_suite.AbstractTestSuite.__init__(self) |
| 40 | self._package_name = None |
| 41 | self._runner_name = self.DEFAULT_RUNNER |
| 42 | self._class_name = None |
| 43 | self._target_name = None |
Brett Chabot | 1cd6174 | 2009-10-21 17:00:58 -0700 | [diff] [blame] | 44 | self._java_package = None |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 45 | |
| 46 | def GetPackageName(self): |
Brett Chabot | 924c089 | 2009-10-21 14:23:54 -0700 | [diff] [blame] | 47 | return self._package_name |
| 48 | |
| 49 | def SetPackageName(self, package_name): |
| 50 | self._package_name = package_name |
| 51 | return self |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 52 | |
| 53 | def GetRunnerName(self): |
Brett Chabot | 924c089 | 2009-10-21 14:23:54 -0700 | [diff] [blame] | 54 | return self._runner_name |
| 55 | |
| 56 | def SetRunnerName(self, runner_name): |
| 57 | self._runner_name = runner_name |
| 58 | return self |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 59 | |
| 60 | def GetClassName(self): |
Brett Chabot | 924c089 | 2009-10-21 14:23:54 -0700 | [diff] [blame] | 61 | return self._class_name |
| 62 | |
| 63 | def SetClassName(self, class_name): |
| 64 | self._class_name = class_name |
| 65 | return self |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 66 | |
Brett Chabot | 1cd6174 | 2009-10-21 17:00:58 -0700 | [diff] [blame] | 67 | def GetJavaPackageFilter(self): |
| 68 | return self._java_package |
| 69 | |
| 70 | def SetJavaPackageFilter(self, java_package_name): |
| 71 | """Configure the suite to only run tests in given java package.""" |
| 72 | self._java_package = java_package_name |
| 73 | return self |
| 74 | |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 75 | def GetTargetName(self): |
| 76 | """Retrieve module that this test is targeting. |
| 77 | |
| 78 | Used for generating code coverage metrics. |
Brett Chabot | 924c089 | 2009-10-21 14:23:54 -0700 | [diff] [blame] | 79 | Returns: |
| 80 | the module target name |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 81 | """ |
Brett Chabot | 924c089 | 2009-10-21 14:23:54 -0700 | [diff] [blame] | 82 | return self._target_name |
| 83 | |
| 84 | def SetTargetName(self, target_name): |
| 85 | self._target_name = target_name |
| 86 | return self |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 87 | |
| 88 | def GetBuildDependencies(self, options): |
| 89 | if options.coverage: |
Guang Zhu | 863870c | 2010-04-15 10:48:26 -0700 | [diff] [blame] | 90 | return [self._LIBCORE_BUILD_PATH] |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 91 | return [] |
| 92 | |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 93 | def Run(self, options, adb): |
| 94 | """Run the provided test suite. |
| 95 | |
| 96 | Builds up an adb instrument command using provided input arguments. |
| 97 | |
| 98 | Args: |
| 99 | options: command line options to provide to test run |
| 100 | adb: adb_interface to device under test |
Brett Chabot | 924c089 | 2009-10-21 14:23:54 -0700 | [diff] [blame] | 101 | |
| 102 | Raises: |
| 103 | errors.AbortError: if fatal error occurs |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 104 | """ |
| 105 | |
| 106 | test_class = self.GetClassName() |
| 107 | if options.test_class is not None: |
| 108 | test_class = options.test_class.lstrip() |
| 109 | if test_class.startswith("."): |
Brett Chabot | 3ae5f8a | 2009-06-28 12:00:47 -0700 | [diff] [blame] | 110 | test_class = self.GetPackageName() + test_class |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 111 | if options.test_method is not None: |
| 112 | test_class = "%s#%s" % (test_class, options.test_method) |
| 113 | |
Brett Chabot | 1cd6174 | 2009-10-21 17:00:58 -0700 | [diff] [blame] | 114 | test_package = self.GetJavaPackageFilter() |
| 115 | if options.test_package: |
| 116 | test_package = options.test_package |
| 117 | |
| 118 | if test_class and test_package: |
| 119 | logger.Log('Error: both class and java package options are specified') |
| 120 | |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 121 | instrumentation_args = {} |
| 122 | if test_class is not None: |
| 123 | instrumentation_args["class"] = test_class |
Brett Chabot | 1cd6174 | 2009-10-21 17:00:58 -0700 | [diff] [blame] | 124 | if test_package: |
| 125 | instrumentation_args["package"] = test_package |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 126 | if options.test_size: |
| 127 | instrumentation_args["size"] = options.test_size |
| 128 | if options.wait_for_debugger: |
| 129 | instrumentation_args["debug"] = "true" |
| 130 | if options.suite_assign_mode: |
| 131 | instrumentation_args["suiteAssignment"] = "true" |
| 132 | if options.coverage: |
| 133 | instrumentation_args["coverage"] = "true" |
Brett Chabot | c061154 | 2010-02-20 20:09:58 -0800 | [diff] [blame] | 134 | if options.test_annotation: |
| 135 | instrumentation_args["annotation"] = options.test_annotation |
| 136 | if options.test_not_annotation: |
| 137 | instrumentation_args["notAnnotation"] = options.test_not_annotation |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 138 | if options.preview: |
| 139 | adb_cmd = adb.PreviewInstrumentationCommand( |
| 140 | package_name=self.GetPackageName(), |
| 141 | runner_name=self.GetRunnerName(), |
| 142 | raw_mode=options.raw_mode, |
| 143 | instrumentation_args=instrumentation_args) |
| 144 | logger.Log(adb_cmd) |
| 145 | elif options.coverage: |
| 146 | coverage_gen = coverage.CoverageGenerator(adb) |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 147 | adb.WaitForInstrumentation(self.GetPackageName(), |
| 148 | self.GetRunnerName()) |
| 149 | # need to parse test output to determine path to coverage file |
| 150 | logger.Log("Running in coverage mode, suppressing test output") |
| 151 | try: |
| 152 | (test_results, status_map) = adb.StartInstrumentationForPackage( |
Brett Chabot | 924c089 | 2009-10-21 14:23:54 -0700 | [diff] [blame] | 153 | package_name=self.GetPackageName(), |
| 154 | runner_name=self.GetRunnerName(), |
| 155 | timeout_time=60*60, |
| 156 | instrumentation_args=instrumentation_args) |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 157 | except errors.InstrumentationError, errors.DeviceUnresponsiveError: |
| 158 | return |
| 159 | self._PrintTestResults(test_results) |
| 160 | device_coverage_path = status_map.get("coverageFilePath", None) |
| 161 | if device_coverage_path is None: |
| 162 | logger.Log("Error: could not find coverage data on device") |
| 163 | return |
| 164 | |
Brett Chabot | 8424ffc | 2010-01-21 17:29:18 -0800 | [diff] [blame] | 165 | coverage_file = coverage_gen.ExtractReport( |
| 166 | self, device_coverage_path, test_qualifier=options.test_size) |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 167 | if coverage_file is not None: |
| 168 | logger.Log("Coverage report generated at %s" % coverage_file) |
| 169 | else: |
| 170 | adb.WaitForInstrumentation(self.GetPackageName(), |
| 171 | self.GetRunnerName()) |
| 172 | adb.StartInstrumentationNoResults( |
| 173 | package_name=self.GetPackageName(), |
| 174 | runner_name=self.GetRunnerName(), |
| 175 | raw_mode=options.raw_mode, |
| 176 | instrumentation_args=instrumentation_args) |
| 177 | |
| 178 | def _PrintTestResults(self, test_results): |
| 179 | """Prints a summary of test result data to stdout. |
| 180 | |
| 181 | Args: |
| 182 | test_results: a list of am_instrument_parser.TestResult |
| 183 | """ |
| 184 | total_count = 0 |
| 185 | error_count = 0 |
| 186 | fail_count = 0 |
| 187 | for test_result in test_results: |
Brett Chabot | 924c089 | 2009-10-21 14:23:54 -0700 | [diff] [blame] | 188 | if test_result.GetStatusCode() == -1: # error |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 189 | logger.Log("Error in %s: %s" % (test_result.GetTestName(), |
| 190 | test_result.GetFailureReason())) |
| 191 | error_count+=1 |
Brett Chabot | 924c089 | 2009-10-21 14:23:54 -0700 | [diff] [blame] | 192 | elif test_result.GetStatusCode() == -2: # failure |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 193 | logger.Log("Failure in %s: %s" % (test_result.GetTestName(), |
| 194 | test_result.GetFailureReason())) |
| 195 | fail_count+=1 |
| 196 | total_count+=1 |
| 197 | logger.Log("Tests run: %d, Failures: %d, Errors: %d" % |
| 198 | (total_count, fail_count, error_count)) |