The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 1 | #!/usr/bin/python2.4 |
| 2 | # |
| 3 | # Copyright 2008, The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
| 17 | """Command line utility for running a pre-defined test. |
| 18 | |
| 19 | Based on previous <androidroot>/development/tools/runtest shell script. |
| 20 | """ |
| 21 | |
| 22 | # Python imports |
| 23 | import glob |
| 24 | import optparse |
| 25 | import os |
Niko Catania | 2e990b9 | 2009-04-02 16:52:26 -0700 | [diff] [blame] | 26 | import re |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 27 | from sets import Set |
| 28 | import sys |
| 29 | |
| 30 | # local imports |
| 31 | import adb_interface |
| 32 | import android_build |
| 33 | import coverage |
| 34 | import errors |
| 35 | import logger |
| 36 | import run_command |
| 37 | import test_defs |
| 38 | |
| 39 | |
| 40 | class TestRunner(object): |
| 41 | """Command line utility class for running pre-defined Android test(s).""" |
| 42 | |
Brett Chabot | f61f43e | 2009-04-02 11:52:48 -0700 | [diff] [blame] | 43 | _TEST_FILE_NAME = "test_defs.xml" |
| 44 | |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 45 | # file path to android core platform tests, relative to android build root |
| 46 | # TODO move these test data files to another directory |
Nicolas Catania | 97b24c4 | 2009-04-22 11:08:32 -0700 | [diff] [blame] | 47 | _CORE_TEST_PATH = os.path.join("development", "testrunner", |
Brett Chabot | f61f43e | 2009-04-02 11:52:48 -0700 | [diff] [blame] | 48 | _TEST_FILE_NAME) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 49 | |
| 50 | # vendor glob file path patterns to tests, relative to android |
| 51 | # build root |
| 52 | _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo", |
Brett Chabot | f61f43e | 2009-04-02 11:52:48 -0700 | [diff] [blame] | 53 | _TEST_FILE_NAME) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 54 | |
| 55 | _RUNTEST_USAGE = ( |
| 56 | "usage: runtest.py [options] short-test-name[s]\n\n" |
| 57 | "The runtest script works in two ways. You can query it " |
| 58 | "for a list of tests, or you can launch one or more tests.") |
| 59 | |
Brett Chabot | 72731f3 | 2009-03-31 11:14:05 -0700 | [diff] [blame] | 60 | def __init__(self): |
| 61 | # disable logging of timestamp |
Niko Catania | 2e990b9 | 2009-04-02 16:52:26 -0700 | [diff] [blame] | 62 | self._root_path = android_build.GetTop() |
Nicolas Catania | 97b24c4 | 2009-04-22 11:08:32 -0700 | [diff] [blame] | 63 | logger.SetTimestampLogging(False) |
Brett Chabot | 72731f3 | 2009-03-31 11:14:05 -0700 | [diff] [blame] | 64 | |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 65 | def _ProcessOptions(self): |
| 66 | """Processes command-line options.""" |
| 67 | # TODO error messages on once-only or mutually-exclusive options. |
| 68 | user_test_default = os.path.join(os.environ.get("HOME"), ".android", |
Brett Chabot | f61f43e | 2009-04-02 11:52:48 -0700 | [diff] [blame] | 69 | self._TEST_FILE_NAME) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 70 | |
| 71 | parser = optparse.OptionParser(usage=self._RUNTEST_USAGE) |
| 72 | |
| 73 | parser.add_option("-l", "--list-tests", dest="only_list_tests", |
| 74 | default=False, action="store_true", |
| 75 | help="To view the list of tests") |
| 76 | parser.add_option("-b", "--skip-build", dest="skip_build", default=False, |
| 77 | action="store_true", help="Skip build - just launch") |
| 78 | parser.add_option("-n", "--skip_execute", dest="preview", default=False, |
| 79 | action="store_true", |
| 80 | help="Do not execute, just preview commands") |
| 81 | parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False, |
| 82 | action="store_true", |
| 83 | help="Raw mode (for output to other tools)") |
| 84 | parser.add_option("-a", "--suite-assign", dest="suite_assign_mode", |
| 85 | default=False, action="store_true", |
| 86 | help="Suite assignment (for details & usage see " |
| 87 | "InstrumentationTestRunner)") |
| 88 | parser.add_option("-v", "--verbose", dest="verbose", default=False, |
| 89 | action="store_true", |
| 90 | help="Increase verbosity of %s" % sys.argv[0]) |
| 91 | parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger", |
| 92 | default=False, action="store_true", |
| 93 | help="Wait for debugger before launching tests") |
| 94 | parser.add_option("-c", "--test-class", dest="test_class", |
| 95 | help="Restrict test to a specific class") |
| 96 | parser.add_option("-m", "--test-method", dest="test_method", |
| 97 | help="Restrict test to a specific method") |
Brett Chabot | 8a101cb | 2009-05-05 12:56:39 -0700 | [diff] [blame] | 98 | parser.add_option("-p", "--test-package", dest="test_package", |
| 99 | help="Restrict test to a specific java package") |
| 100 | parser.add_option("-z", "--size", dest="test_size", |
| 101 | help="Restrict test to a specific test size") |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 102 | parser.add_option("-u", "--user-tests-file", dest="user_tests_file", |
| 103 | metavar="FILE", default=user_test_default, |
| 104 | help="Alternate source of user test definitions") |
| 105 | parser.add_option("-o", "--coverage", dest="coverage", |
| 106 | default=False, action="store_true", |
| 107 | help="Generate code coverage metrics for test(s)") |
| 108 | parser.add_option("-t", "--all-tests", dest="all_tests", |
| 109 | default=False, action="store_true", |
| 110 | help="Run all defined tests") |
| 111 | parser.add_option("--continuous", dest="continuous_tests", |
| 112 | default=False, action="store_true", |
| 113 | help="Run all tests defined as part of the continuous " |
| 114 | "test set") |
Wei-Ta Chen | 97752d4 | 2009-05-21 16:24:04 -0700 | [diff] [blame] | 115 | parser.add_option("--timeout", dest="timeout", |
| 116 | default=300, help="Set a timeout limit (in sec) for " |
| 117 | "running native tests on a device (default: 300 secs)") |
Brett Chabot | 49b7711 | 2009-06-02 11:46:04 -0700 | [diff] [blame^] | 118 | parser.add_option("--cts", dest="cts_tests", |
| 119 | default=False, action="store_true", |
| 120 | help="Run all tests defined as part of the " |
| 121 | "compatibility test suite") |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 122 | group = optparse.OptionGroup( |
| 123 | parser, "Targets", "Use these options to direct tests to a specific " |
| 124 | "Android target") |
| 125 | group.add_option("-e", "--emulator", dest="emulator", default=False, |
| 126 | action="store_true", help="use emulator") |
| 127 | group.add_option("-d", "--device", dest="device", default=False, |
| 128 | action="store_true", help="use device") |
| 129 | group.add_option("-s", "--serial", dest="serial", |
| 130 | help="use specific serial") |
| 131 | parser.add_option_group(group) |
| 132 | |
| 133 | self._options, self._test_args = parser.parse_args() |
| 134 | |
Brett Chabot | 49b7711 | 2009-06-02 11:46:04 -0700 | [diff] [blame^] | 135 | if (not self._options.only_list_tests |
| 136 | and not self._options.all_tests |
| 137 | and not self._options.continuous_tests |
| 138 | and not self._options.cts_tests |
| 139 | and len(self._test_args) < 1): |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 140 | parser.print_help() |
| 141 | logger.SilentLog("at least one test name must be specified") |
| 142 | raise errors.AbortError |
| 143 | |
| 144 | self._adb = adb_interface.AdbInterface() |
| 145 | if self._options.emulator: |
| 146 | self._adb.SetEmulatorTarget() |
| 147 | elif self._options.device: |
| 148 | self._adb.SetDeviceTarget() |
| 149 | elif self._options.serial is not None: |
| 150 | self._adb.SetTargetSerial(self._options.serial) |
| 151 | |
| 152 | if self._options.verbose: |
| 153 | logger.SetVerbose(True) |
| 154 | |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 155 | self._known_tests = self._ReadTests() |
| 156 | |
| 157 | self._coverage_gen = coverage.CoverageGenerator( |
| 158 | android_root_path=self._root_path, adb_interface=self._adb) |
| 159 | |
| 160 | def _ReadTests(self): |
| 161 | """Parses the set of test definition data. |
| 162 | |
| 163 | Returns: |
| 164 | A TestDefinitions object that contains the set of parsed tests. |
| 165 | Raises: |
| 166 | AbortError: If a fatal error occurred when parsing the tests. |
| 167 | """ |
| 168 | core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH) |
| 169 | try: |
| 170 | known_tests = test_defs.TestDefinitions() |
| 171 | known_tests.Parse(core_test_path) |
Brett Chabot | 2d85c0e | 2009-03-31 15:19:13 -0700 | [diff] [blame] | 172 | # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 173 | vendor_tests_pattern = os.path.join(self._root_path, |
| 174 | self._VENDOR_TEST_PATH) |
| 175 | test_file_paths = glob.glob(vendor_tests_pattern) |
| 176 | for test_file_path in test_file_paths: |
| 177 | known_tests.Parse(test_file_path) |
| 178 | if os.path.isfile(self._options.user_tests_file): |
| 179 | known_tests.Parse(self._options.user_tests_file) |
| 180 | return known_tests |
| 181 | except errors.ParseError: |
| 182 | raise errors.AbortError |
| 183 | |
| 184 | def _DumpTests(self): |
| 185 | """Prints out set of defined tests.""" |
| 186 | print "The following tests are currently defined:" |
| 187 | for test in self._known_tests: |
Niko Catania | 2e990b9 | 2009-04-02 16:52:26 -0700 | [diff] [blame] | 188 | print "%-15s %s" % (test.GetName(), test.GetDescription()) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 189 | |
| 190 | def _DoBuild(self): |
| 191 | logger.SilentLog("Building tests...") |
| 192 | target_set = Set() |
Niko Catania | a6dc2ab | 2009-04-03 14:12:46 -0700 | [diff] [blame] | 193 | extra_args_set = Set() |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 194 | for test_suite in self._GetTestsToRun(): |
Niko Catania | a6dc2ab | 2009-04-03 14:12:46 -0700 | [diff] [blame] | 195 | self._AddBuildTarget(test_suite, target_set, extra_args_set) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 196 | |
| 197 | if target_set: |
| 198 | if self._options.coverage: |
| 199 | self._coverage_gen.EnableCoverageBuild() |
Brett Chabot | 2b6643b | 2009-04-07 18:35:27 -0700 | [diff] [blame] | 200 | self._AddBuildTargetPath(self._coverage_gen.GetEmmaBuildPath(), |
| 201 | target_set) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 202 | target_build_string = " ".join(list(target_set)) |
Niko Catania | a6dc2ab | 2009-04-03 14:12:46 -0700 | [diff] [blame] | 203 | extra_args_string = " ".join(list(extra_args_set)) |
Brett Chabot | 2b6643b | 2009-04-07 18:35:27 -0700 | [diff] [blame] | 204 | # log the user-friendly equivalent make command, so developers can |
| 205 | # replicate this step |
| 206 | logger.Log("mmm %s %s" % (target_build_string, extra_args_string)) |
| 207 | # mmm cannot be used from python, so perform a similiar operation using |
| 208 | # ONE_SHOT_MAKEFILE |
Niko Catania | a6dc2ab | 2009-04-03 14:12:46 -0700 | [diff] [blame] | 209 | cmd = 'ONE_SHOT_MAKEFILE="%s" make -C "%s" files %s' % ( |
| 210 | target_build_string, self._root_path, extra_args_string) |
Niko Catania | a6dc2ab | 2009-04-03 14:12:46 -0700 | [diff] [blame] | 211 | |
Brett Chabot | 72731f3 | 2009-03-31 11:14:05 -0700 | [diff] [blame] | 212 | if self._options.preview: |
| 213 | # in preview mode, just display to the user what command would have been |
| 214 | # run |
| 215 | logger.Log("adb sync") |
| 216 | else: |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 217 | run_command.RunCommand(cmd, return_output=False) |
| 218 | logger.Log("Syncing to device...") |
| 219 | self._adb.Sync() |
| 220 | |
Niko Catania | a6dc2ab | 2009-04-03 14:12:46 -0700 | [diff] [blame] | 221 | def _AddBuildTarget(self, test_suite, target_set, extra_args_set): |
| 222 | build_dir = test_suite.GetBuildPath() |
Brett Chabot | 2b6643b | 2009-04-07 18:35:27 -0700 | [diff] [blame] | 223 | if self._AddBuildTargetPath(build_dir, target_set): |
| 224 | extra_args_set.add(test_suite.GetExtraMakeArgs()) |
| 225 | |
| 226 | def _AddBuildTargetPath(self, build_dir, target_set): |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 227 | if build_dir is not None: |
| 228 | build_file_path = os.path.join(build_dir, "Android.mk") |
| 229 | if os.path.isfile(os.path.join(self._root_path, build_file_path)): |
| 230 | target_set.add(build_file_path) |
Brett Chabot | 2b6643b | 2009-04-07 18:35:27 -0700 | [diff] [blame] | 231 | return True |
| 232 | return False |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 233 | |
| 234 | def _GetTestsToRun(self): |
| 235 | """Get a list of TestSuite objects to run, based on command line args.""" |
| 236 | if self._options.all_tests: |
| 237 | return self._known_tests.GetTests() |
Brett Chabot | 49b7711 | 2009-06-02 11:46:04 -0700 | [diff] [blame^] | 238 | elif self._options.continuous_tests: |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 239 | return self._known_tests.GetContinuousTests() |
Brett Chabot | 49b7711 | 2009-06-02 11:46:04 -0700 | [diff] [blame^] | 240 | elif self._options.cts_tests: |
| 241 | return self._known_tests.GetCtsTests() |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 242 | tests = [] |
| 243 | for name in self._test_args: |
| 244 | test = self._known_tests.GetTest(name) |
| 245 | if test is None: |
| 246 | logger.Log("Error: Could not find test %s" % name) |
| 247 | self._DumpTests() |
| 248 | raise errors.AbortError |
| 249 | tests.append(test) |
| 250 | return tests |
| 251 | |
| 252 | def _RunTest(self, test_suite): |
| 253 | """Run the provided test suite. |
| 254 | |
| 255 | Builds up an adb instrument command using provided input arguments. |
| 256 | |
| 257 | Args: |
| 258 | test_suite: TestSuite to run |
| 259 | """ |
| 260 | |
| 261 | test_class = test_suite.GetClassName() |
| 262 | if self._options.test_class is not None: |
Brett Chabot | 292df41 | 2009-05-07 11:09:40 -0700 | [diff] [blame] | 263 | test_class = self._options.test_class.lstrip() |
| 264 | if test_class.startswith("."): |
| 265 | test_class = test_suite.GetPackageName() + test_class |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 266 | if self._options.test_method is not None: |
| 267 | test_class = "%s#%s" % (test_class, self._options.test_method) |
| 268 | |
| 269 | instrumentation_args = {} |
| 270 | if test_class is not None: |
| 271 | instrumentation_args["class"] = test_class |
Brett Chabot | 8a101cb | 2009-05-05 12:56:39 -0700 | [diff] [blame] | 272 | if self._options.test_package: |
| 273 | instrumentation_args["package"] = self._options.test_package |
| 274 | if self._options.test_size: |
| 275 | instrumentation_args["size"] = self._options.test_size |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 276 | if self._options.wait_for_debugger: |
| 277 | instrumentation_args["debug"] = "true" |
| 278 | if self._options.suite_assign_mode: |
| 279 | instrumentation_args["suiteAssignment"] = "true" |
| 280 | if self._options.coverage: |
| 281 | instrumentation_args["coverage"] = "true" |
| 282 | if self._options.preview: |
| 283 | adb_cmd = self._adb.PreviewInstrumentationCommand( |
| 284 | package_name=test_suite.GetPackageName(), |
| 285 | runner_name=test_suite.GetRunnerName(), |
| 286 | raw_mode=self._options.raw_mode, |
| 287 | instrumentation_args=instrumentation_args) |
| 288 | logger.Log(adb_cmd) |
Brett Chabot | ae68f1a | 2009-05-28 18:29:24 -0700 | [diff] [blame] | 289 | elif self._options.coverage: |
| 290 | # need to parse test output to determine path to coverage file |
| 291 | logger.Log("Running in coverage mode, suppressing test output") |
| 292 | try: |
| 293 | (test_results, status_map) = self._adb.StartInstrumentationForPackage( |
| 294 | package_name=test_suite.GetPackageName(), |
| 295 | runner_name=test_suite.GetRunnerName(), |
| 296 | timeout_time=60*60, |
| 297 | instrumentation_args=instrumentation_args) |
| 298 | except errors.InstrumentationError, errors.DeviceUnresponsiveError: |
| 299 | return |
| 300 | self._PrintTestResults(test_results) |
| 301 | device_coverage_path = status_map.get("coverageFilePath", None) |
| 302 | if device_coverage_path is None: |
| 303 | logger.Log("Error: could not find coverage data on device") |
| 304 | return |
| 305 | coverage_file = self._coverage_gen.ExtractReport(test_suite, device_coverage_path) |
| 306 | if coverage_file is not None: |
| 307 | logger.Log("Coverage report generated at %s" % coverage_file) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 308 | else: |
| 309 | self._adb.StartInstrumentationNoResults( |
| 310 | package_name=test_suite.GetPackageName(), |
| 311 | runner_name=test_suite.GetRunnerName(), |
| 312 | raw_mode=self._options.raw_mode, |
| 313 | instrumentation_args=instrumentation_args) |
Brett Chabot | ae68f1a | 2009-05-28 18:29:24 -0700 | [diff] [blame] | 314 | |
| 315 | def _PrintTestResults(self, test_results): |
| 316 | """Prints a summary of test result data to stdout. |
| 317 | |
| 318 | Args: |
| 319 | test_results: a list of am_instrument_parser.TestResult |
| 320 | """ |
| 321 | total_count = 0 |
| 322 | error_count = 0 |
| 323 | fail_count = 0 |
| 324 | for test_result in test_results: |
| 325 | if test_result.GetStatusCode() == -1: # error |
| 326 | logger.Log("Error in %s: %s" % (test_result.GetTestName(), |
| 327 | test_result.GetFailureReason())) |
| 328 | error_count+=1 |
| 329 | elif test_result.GetStatusCode() == -2: # failure |
| 330 | logger.Log("Failure in %s: %s" % (test_result.GetTestName(), |
| 331 | test_result.GetFailureReason())) |
| 332 | fail_count+=1 |
| 333 | total_count+=1 |
| 334 | logger.Log("Tests run: %d, Failures: %d, Errors: %d" % |
| 335 | (total_count, fail_count, error_count)) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 336 | |
Nicolas Catania | ff096c1 | 2009-05-01 11:55:36 -0700 | [diff] [blame] | 337 | def _CollectTestSources(self, test_list, dirname, files): |
| 338 | """For each directory, find tests source file and add them to the list. |
| 339 | |
| 340 | Test files must match one of the following pattern: |
| 341 | - test_*.[cc|cpp] |
| 342 | - *_test.[cc|cpp] |
| 343 | - *_unittest.[cc|cpp] |
| 344 | |
| 345 | This method is a callback for os.path.walk. |
| 346 | |
| 347 | Args: |
| 348 | test_list: Where new tests should be inserted. |
| 349 | dirname: Current directory. |
| 350 | files: List of files in the current directory. |
| 351 | """ |
| 352 | for f in files: |
| 353 | (name, ext) = os.path.splitext(f) |
| 354 | if ext == ".cc" or ext == ".cpp": |
| 355 | if re.search("_test$|_test_$|_unittest$|_unittest_$|^test_", name): |
| 356 | logger.SilentLog("Found %s" % f) |
| 357 | test_list.append(str(os.path.join(dirname, f))) |
| 358 | |
| 359 | def _FilterOutMissing(self, path, sources): |
| 360 | """Filter out from the sources list missing tests. |
| 361 | |
| 362 | Sometimes some test source are not built for the target, i.e there |
| 363 | is no binary corresponding to the source file. We need to filter |
| 364 | these out. |
| 365 | |
| 366 | Args: |
| 367 | path: Where the binaries should be. |
| 368 | sources: List of tests source path. |
| 369 | Returns: |
| 370 | A list of test binaries built from the sources. |
| 371 | """ |
| 372 | binaries = [] |
| 373 | for f in sources: |
| 374 | binary = os.path.basename(f) |
| 375 | binary = os.path.splitext(binary)[0] |
| 376 | full_path = os.path.join(path, binary) |
| 377 | if os.path.exists(full_path): |
| 378 | binaries.append(binary) |
| 379 | return binaries |
| 380 | |
Niko Catania | 2e990b9 | 2009-04-02 16:52:26 -0700 | [diff] [blame] | 381 | def _RunNativeTest(self, test_suite): |
| 382 | """Run the provided *native* test suite. |
| 383 | |
Nicolas Catania | ff096c1 | 2009-05-01 11:55:36 -0700 | [diff] [blame] | 384 | The test_suite must contain a build path where the native test |
| 385 | files are. Subdirectories are automatically scanned as well. |
| 386 | |
| 387 | Each test's name must have a .cc or .cpp extension and match one |
| 388 | of the following patterns: |
| 389 | - test_* |
| 390 | - *_test.[cc|cpp] |
| 391 | - *_unittest.[cc|cpp] |
Niko Catania | 2e990b9 | 2009-04-02 16:52:26 -0700 | [diff] [blame] | 392 | A successful test must return 0. Any other value will be considered |
| 393 | as an error. |
| 394 | |
| 395 | Args: |
| 396 | test_suite: TestSuite to run |
| 397 | """ |
| 398 | # find all test files, convert unicode names to ascii, take the basename |
| 399 | # and drop the .cc/.cpp extension. |
Nicolas Catania | ff096c1 | 2009-05-01 11:55:36 -0700 | [diff] [blame] | 400 | source_list = [] |
| 401 | build_path = test_suite.GetBuildPath() |
| 402 | os.path.walk(build_path, self._CollectTestSources, source_list) |
| 403 | logger.SilentLog("Tests source %s" % source_list) |
| 404 | |
| 405 | # Host tests are under out/host/<os>-<arch>/bin. |
| 406 | host_list = self._FilterOutMissing(android_build.GetHostBin(), source_list) |
| 407 | logger.SilentLog("Host tests %s" % host_list) |
| 408 | |
| 409 | # Target tests are under $ANDROID_PRODUCT_OUT/system/bin. |
| 410 | target_list = self._FilterOutMissing(android_build.GetTargetSystemBin(), |
| 411 | source_list) |
| 412 | logger.SilentLog("Target tests %s" % target_list) |
Niko Catania | 2e990b9 | 2009-04-02 16:52:26 -0700 | [diff] [blame] | 413 | |
Nicolas Catania | 97b24c4 | 2009-04-22 11:08:32 -0700 | [diff] [blame] | 414 | # Run on the host |
| 415 | logger.Log("\nRunning on host") |
Nicolas Catania | ff096c1 | 2009-05-01 11:55:36 -0700 | [diff] [blame] | 416 | for f in host_list: |
Nicolas Catania | 97b24c4 | 2009-04-22 11:08:32 -0700 | [diff] [blame] | 417 | if run_command.RunHostCommand(f) != 0: |
| 418 | logger.Log("%s... failed" % f) |
| 419 | else: |
Nicolas Catania | bcd93dc | 2009-05-14 12:25:23 -0700 | [diff] [blame] | 420 | if run_command.HasValgrind(): |
| 421 | if run_command.RunHostCommand(f, valgrind=True) == 0: |
| 422 | logger.Log("%s... ok\t\t[valgrind: ok]" % f) |
| 423 | else: |
| 424 | logger.Log("%s... ok\t\t[valgrind: failed]" % f) |
Nicolas Catania | 97b24c4 | 2009-04-22 11:08:32 -0700 | [diff] [blame] | 425 | else: |
Nicolas Catania | bcd93dc | 2009-05-14 12:25:23 -0700 | [diff] [blame] | 426 | logger.Log("%s... ok\t\t[valgrind: missing]" % f) |
Nicolas Catania | 97b24c4 | 2009-04-22 11:08:32 -0700 | [diff] [blame] | 427 | |
| 428 | # Run on the device |
| 429 | logger.Log("\nRunning on target") |
Nicolas Catania | ff096c1 | 2009-05-01 11:55:36 -0700 | [diff] [blame] | 430 | for f in target_list: |
| 431 | full_path = os.path.join(os.sep, "system", "bin", f) |
Niko Catania | 2e990b9 | 2009-04-02 16:52:26 -0700 | [diff] [blame] | 432 | |
Niko Catania | fa14bd5 | 2009-04-09 16:50:54 -0700 | [diff] [blame] | 433 | # Single quotes are needed to prevent the shell splitting it. |
Nicolas Catania | bcd93dc | 2009-05-14 12:25:23 -0700 | [diff] [blame] | 434 | output = self._adb.SendShellCommand("'%s 2>&1;echo -n exit code:$?'" % |
Wei-Ta Chen | 97752d4 | 2009-05-21 16:24:04 -0700 | [diff] [blame] | 435 | full_path, |
| 436 | int(self._options.timeout)) |
Nicolas Catania | bcd93dc | 2009-05-14 12:25:23 -0700 | [diff] [blame] | 437 | success = output.endswith("exit code:0") |
| 438 | logger.Log("%s... %s" % (f, success and "ok" or "failed")) |
| 439 | # Print the captured output when the test failed. |
| 440 | if not success or self._options.verbose: |
| 441 | pos = output.rfind("exit code") |
| 442 | output = output[0:pos] |
| 443 | logger.Log(output) |
Niko Catania | 2e990b9 | 2009-04-02 16:52:26 -0700 | [diff] [blame] | 444 | |
| 445 | # Cleanup |
| 446 | self._adb.SendShellCommand("rm %s" % full_path) |
| 447 | |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 448 | def RunTests(self): |
| 449 | """Main entry method - executes the tests according to command line args.""" |
| 450 | try: |
| 451 | run_command.SetAbortOnError() |
| 452 | self._ProcessOptions() |
| 453 | if self._options.only_list_tests: |
| 454 | self._DumpTests() |
| 455 | return |
| 456 | |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 457 | if not self._options.skip_build: |
| 458 | self._DoBuild() |
| 459 | |
| 460 | for test_suite in self._GetTestsToRun(): |
Niko Catania | 2e990b9 | 2009-04-02 16:52:26 -0700 | [diff] [blame] | 461 | if test_suite.IsNative(): |
| 462 | self._RunNativeTest(test_suite) |
| 463 | else: |
| 464 | self._RunTest(test_suite) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 465 | except KeyboardInterrupt: |
| 466 | logger.Log("Exiting...") |
Brett Chabot | 8a101cb | 2009-05-05 12:56:39 -0700 | [diff] [blame] | 467 | except errors.AbortError, e: |
| 468 | logger.Log(e.msg) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 469 | logger.SilentLog("Exiting due to AbortError...") |
| 470 | except errors.WaitForResponseTimedOutError: |
| 471 | logger.Log("Timed out waiting for response") |
| 472 | |
| 473 | |
| 474 | def RunTests(): |
| 475 | runner = TestRunner() |
| 476 | runner.RunTests() |
| 477 | |
| 478 | if __name__ == "__main__": |
| 479 | RunTests() |