Geremy Condra | 01844b1 | 2012-09-11 15:24:25 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 2 | # |
| 3 | # Copyright 2008, The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
Brett Chabot | 59b4778 | 2009-10-21 17:23:01 -0700 | [diff] [blame] | 17 | """Command line utility for running Android tests |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 18 | |
Brett Chabot | 59b4778 | 2009-10-21 17:23:01 -0700 | [diff] [blame] | 19 | runtest helps automate the instructions for building and running tests |
| 20 | - It builds the corresponding test package for the code you want to test |
| 21 | - It pushes the test package to your device or emulator |
| 22 | - It launches InstrumentationTestRunner (or similar) to run the tests you |
| 23 | specify. |
| 24 | |
| 25 | runtest supports running tests whose attributes have been pre-defined in |
| 26 | _TEST_FILE_NAME files, (runtest <testname>), or by specifying the file |
| 27 | system path to the test to run (runtest --path <path>). |
| 28 | |
| 29 | Do runtest --help to see full list of options. |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 30 | """ |
| 31 | |
| 32 | # Python imports |
| 33 | import glob |
| 34 | import optparse |
| 35 | import os |
Brett Chabot | 7454171 | 2012-08-31 18:39:00 -0700 | [diff] [blame] | 36 | import re |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 37 | from sets import Set |
| 38 | import sys |
Brett Chabot | cdfaae1 | 2011-06-07 10:10:38 -0700 | [diff] [blame] | 39 | import time |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 40 | |
| 41 | # local imports |
| 42 | import adb_interface |
| 43 | import android_build |
Brett Chabot | 8ac5118 | 2012-09-19 07:35:35 -0700 | [diff] [blame] | 44 | from coverage import coverage |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 45 | import errors |
| 46 | import logger |
Brett Chabot | 8ac5118 | 2012-09-19 07:35:35 -0700 | [diff] [blame] | 47 | import make_tree |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 48 | import run_command |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 49 | from test_defs import test_defs |
Brett Chabot | 59b4778 | 2009-10-21 17:23:01 -0700 | [diff] [blame] | 50 | from test_defs import test_walker |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 51 | |
| 52 | |
| 53 | class TestRunner(object): |
| 54 | """Command line utility class for running pre-defined Android test(s).""" |
| 55 | |
Brett Chabot | f61f43e | 2009-04-02 11:52:48 -0700 | [diff] [blame] | 56 | _TEST_FILE_NAME = "test_defs.xml" |
| 57 | |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 58 | # file path to android core platform tests, relative to android build root |
| 59 | # TODO move these test data files to another directory |
Nicolas Catania | 97b24c4 | 2009-04-22 11:08:32 -0700 | [diff] [blame] | 60 | _CORE_TEST_PATH = os.path.join("development", "testrunner", |
Brett Chabot | f61f43e | 2009-04-02 11:52:48 -0700 | [diff] [blame] | 61 | _TEST_FILE_NAME) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 62 | |
| 63 | # vendor glob file path patterns to tests, relative to android |
| 64 | # build root |
| 65 | _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo", |
Brett Chabot | f61f43e | 2009-04-02 11:52:48 -0700 | [diff] [blame] | 66 | _TEST_FILE_NAME) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 67 | |
| 68 | _RUNTEST_USAGE = ( |
| 69 | "usage: runtest.py [options] short-test-name[s]\n\n" |
| 70 | "The runtest script works in two ways. You can query it " |
| 71 | "for a list of tests, or you can launch one or more tests.") |
| 72 | |
Brett Chabot | 2477b38 | 2009-09-23 18:05:28 -0700 | [diff] [blame] | 73 | # default value for make -jX |
Brett Chabot | 12db436 | 2012-01-17 16:03:49 -0800 | [diff] [blame] | 74 | _DEFAULT_JOBS = 16 |
Brett Chabot | 2477b38 | 2009-09-23 18:05:28 -0700 | [diff] [blame] | 75 | |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 76 | _DALVIK_VERIFIER_PROP = "dalvik.vm.dexopt-flags" |
| 77 | _DALVIK_VERIFIER_OFF_VALUE = "v=n" |
| 78 | _DALVIK_VERIFIER_OFF_PROP = "%s = %s" %(_DALVIK_VERIFIER_PROP, _DALVIK_VERIFIER_OFF_VALUE) |
Brett Chabot | ccae47d | 2010-06-14 15:19:25 -0700 | [diff] [blame] | 79 | |
Brett Chabot | 5f5928c | 2013-08-20 17:06:03 -0700 | [diff] [blame] | 80 | # regular expression to match path to artifacts to install in make output |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 81 | _RE_MAKE_INSTALL = re.compile(r'INSTALL-PATH:\s([^\s]+)\s(.*)$') |
Brett Chabot | 7454171 | 2012-08-31 18:39:00 -0700 | [diff] [blame] | 82 | |
Brett Chabot | 72731f3 | 2009-03-31 11:14:05 -0700 | [diff] [blame] | 83 | def __init__(self): |
| 84 | # disable logging of timestamp |
Niko Catania | 2e990b9 | 2009-04-02 16:52:26 -0700 | [diff] [blame] | 85 | self._root_path = android_build.GetTop() |
JP Abgrall | f38107c | 2013-07-11 17:39:16 -0700 | [diff] [blame] | 86 | out_base_name = os.path.basename(android_build.GetOutDir()) |
| 87 | # regular expression to find remote device path from a file path relative |
| 88 | # to build root |
| 89 | pattern = r'' + out_base_name + r'\/target\/product\/\w+\/(.+)$' |
| 90 | self._re_make_install_path = re.compile(pattern) |
Nicolas Catania | 97b24c4 | 2009-04-22 11:08:32 -0700 | [diff] [blame] | 91 | logger.SetTimestampLogging(False) |
Brett Chabot | 3ae5f8a | 2009-06-28 12:00:47 -0700 | [diff] [blame] | 92 | self._adb = None |
| 93 | self._known_tests = None |
| 94 | self._options = None |
| 95 | self._test_args = None |
Brett Chabot | 59b4778 | 2009-10-21 17:23:01 -0700 | [diff] [blame] | 96 | self._tests_to_run = None |
Brett Chabot | 72731f3 | 2009-03-31 11:14:05 -0700 | [diff] [blame] | 97 | |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 98 | def _ProcessOptions(self): |
| 99 | """Processes command-line options.""" |
| 100 | # TODO error messages on once-only or mutually-exclusive options. |
| 101 | user_test_default = os.path.join(os.environ.get("HOME"), ".android", |
Brett Chabot | f61f43e | 2009-04-02 11:52:48 -0700 | [diff] [blame] | 102 | self._TEST_FILE_NAME) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 103 | |
| 104 | parser = optparse.OptionParser(usage=self._RUNTEST_USAGE) |
| 105 | |
| 106 | parser.add_option("-l", "--list-tests", dest="only_list_tests", |
| 107 | default=False, action="store_true", |
| 108 | help="To view the list of tests") |
| 109 | parser.add_option("-b", "--skip-build", dest="skip_build", default=False, |
| 110 | action="store_true", help="Skip build - just launch") |
Brett Chabot | 2477b38 | 2009-09-23 18:05:28 -0700 | [diff] [blame] | 111 | parser.add_option("-j", "--jobs", dest="make_jobs", |
| 112 | metavar="X", default=self._DEFAULT_JOBS, |
| 113 | help="Number of make jobs to use when building") |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 114 | parser.add_option("-n", "--skip_execute", dest="preview", default=False, |
| 115 | action="store_true", |
| 116 | help="Do not execute, just preview commands") |
Igor Murashkin | 8d70353 | 2014-01-23 16:13:29 -0800 | [diff] [blame] | 117 | parser.add_option("-i", "--build-install-only", dest="build_install_only", default=False, |
| 118 | action="store_true", |
| 119 | help="Do not execute, build tests and install to device only") |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 120 | parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False, |
| 121 | action="store_true", |
| 122 | help="Raw mode (for output to other tools)") |
| 123 | parser.add_option("-a", "--suite-assign", dest="suite_assign_mode", |
| 124 | default=False, action="store_true", |
| 125 | help="Suite assignment (for details & usage see " |
| 126 | "InstrumentationTestRunner)") |
| 127 | parser.add_option("-v", "--verbose", dest="verbose", default=False, |
| 128 | action="store_true", |
| 129 | help="Increase verbosity of %s" % sys.argv[0]) |
| 130 | parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger", |
| 131 | default=False, action="store_true", |
| 132 | help="Wait for debugger before launching tests") |
| 133 | parser.add_option("-c", "--test-class", dest="test_class", |
| 134 | help="Restrict test to a specific class") |
| 135 | parser.add_option("-m", "--test-method", dest="test_method", |
| 136 | help="Restrict test to a specific method") |
Brett Chabot | 8a101cb | 2009-05-05 12:56:39 -0700 | [diff] [blame] | 137 | parser.add_option("-p", "--test-package", dest="test_package", |
| 138 | help="Restrict test to a specific java package") |
| 139 | parser.add_option("-z", "--size", dest="test_size", |
| 140 | help="Restrict test to a specific test size") |
Brett Chabot | c061154 | 2010-02-20 20:09:58 -0800 | [diff] [blame] | 141 | parser.add_option("--annotation", dest="test_annotation", |
| 142 | help="Include only those tests tagged with a specific" |
| 143 | " annotation") |
| 144 | parser.add_option("--not-annotation", dest="test_not_annotation", |
Brett Chabot | 2e16fbc | 2010-02-23 12:28:27 -0800 | [diff] [blame] | 145 | help="Exclude any tests tagged with a specific" |
Brett Chabot | c061154 | 2010-02-20 20:09:58 -0800 | [diff] [blame] | 146 | " annotation") |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 147 | parser.add_option("-u", "--user-tests-file", dest="user_tests_file", |
| 148 | metavar="FILE", default=user_test_default, |
| 149 | help="Alternate source of user test definitions") |
| 150 | parser.add_option("-o", "--coverage", dest="coverage", |
| 151 | default=False, action="store_true", |
| 152 | help="Generate code coverage metrics for test(s)") |
Brett Chabot | 8ac5118 | 2012-09-19 07:35:35 -0700 | [diff] [blame] | 153 | parser.add_option("--coverage-target", dest="coverage_target_path", |
| 154 | default=None, |
| 155 | help="Path to app to collect code coverage target data for.") |
Santos Cordon | 4e0ad8f | 2015-05-21 12:25:05 -0700 | [diff] [blame] | 156 | parser.add_option("-k", "--skip-permissions", dest="skip_permissions", |
| 157 | default=False, action="store_true", |
| 158 | help="Do not grant runtime permissions during test package" |
| 159 | " installation.") |
Brett Chabot | 59b4778 | 2009-10-21 17:23:01 -0700 | [diff] [blame] | 160 | parser.add_option("-x", "--path", dest="test_path", |
| 161 | help="Run test(s) at given file system path") |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 162 | parser.add_option("-t", "--all-tests", dest="all_tests", |
| 163 | default=False, action="store_true", |
| 164 | help="Run all defined tests") |
| 165 | parser.add_option("--continuous", dest="continuous_tests", |
| 166 | default=False, action="store_true", |
| 167 | help="Run all tests defined as part of the continuous " |
| 168 | "test set") |
Wei-Ta Chen | 97752d4 | 2009-05-21 16:24:04 -0700 | [diff] [blame] | 169 | parser.add_option("--timeout", dest="timeout", |
| 170 | default=300, help="Set a timeout limit (in sec) for " |
| 171 | "running native tests on a device (default: 300 secs)") |
Brett Chabot | 4a5d9f1 | 2010-02-18 20:01:11 -0800 | [diff] [blame] | 172 | parser.add_option("--suite", dest="suite", |
Brett Chabot | 49b7711 | 2009-06-02 11:46:04 -0700 | [diff] [blame] | 173 | help="Run all tests defined as part of the " |
Brett Chabot | 4a5d9f1 | 2010-02-18 20:01:11 -0800 | [diff] [blame] | 174 | "the given test suite") |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 175 | group = optparse.OptionGroup( |
| 176 | parser, "Targets", "Use these options to direct tests to a specific " |
| 177 | "Android target") |
| 178 | group.add_option("-e", "--emulator", dest="emulator", default=False, |
| 179 | action="store_true", help="use emulator") |
| 180 | group.add_option("-d", "--device", dest="device", default=False, |
| 181 | action="store_true", help="use device") |
| 182 | group.add_option("-s", "--serial", dest="serial", |
| 183 | help="use specific serial") |
| 184 | parser.add_option_group(group) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 185 | self._options, self._test_args = parser.parse_args() |
| 186 | |
Brett Chabot | 49b7711 | 2009-06-02 11:46:04 -0700 | [diff] [blame] | 187 | if (not self._options.only_list_tests |
| 188 | and not self._options.all_tests |
| 189 | and not self._options.continuous_tests |
Brett Chabot | 4a5d9f1 | 2010-02-18 20:01:11 -0800 | [diff] [blame] | 190 | and not self._options.suite |
Brett Chabot | 59b4778 | 2009-10-21 17:23:01 -0700 | [diff] [blame] | 191 | and not self._options.test_path |
Brett Chabot | 49b7711 | 2009-06-02 11:46:04 -0700 | [diff] [blame] | 192 | and len(self._test_args) < 1): |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 193 | parser.print_help() |
| 194 | logger.SilentLog("at least one test name must be specified") |
| 195 | raise errors.AbortError |
| 196 | |
| 197 | self._adb = adb_interface.AdbInterface() |
| 198 | if self._options.emulator: |
| 199 | self._adb.SetEmulatorTarget() |
| 200 | elif self._options.device: |
| 201 | self._adb.SetDeviceTarget() |
| 202 | elif self._options.serial is not None: |
| 203 | self._adb.SetTargetSerial(self._options.serial) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 204 | if self._options.verbose: |
| 205 | logger.SetVerbose(True) |
| 206 | |
Brett Chabot | 8ac5118 | 2012-09-19 07:35:35 -0700 | [diff] [blame] | 207 | if self._options.coverage_target_path: |
| 208 | self._options.coverage = True |
| 209 | |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 210 | self._known_tests = self._ReadTests() |
| 211 | |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 212 | self._options.host_lib_path = android_build.GetHostLibraryPath() |
| 213 | self._options.test_data_path = android_build.GetTestAppPath() |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 214 | |
| 215 | def _ReadTests(self): |
| 216 | """Parses the set of test definition data. |
| 217 | |
| 218 | Returns: |
| 219 | A TestDefinitions object that contains the set of parsed tests. |
| 220 | Raises: |
| 221 | AbortError: If a fatal error occurred when parsing the tests. |
| 222 | """ |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 223 | try: |
| 224 | known_tests = test_defs.TestDefinitions() |
Brett Chabot | 3c9cefc | 2011-06-06 20:53:56 -0700 | [diff] [blame] | 225 | # only read tests when not in path mode |
| 226 | if not self._options.test_path: |
| 227 | core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH) |
| 228 | if os.path.isfile(core_test_path): |
| 229 | known_tests.Parse(core_test_path) |
| 230 | # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths |
| 231 | vendor_tests_pattern = os.path.join(self._root_path, |
| 232 | self._VENDOR_TEST_PATH) |
| 233 | test_file_paths = glob.glob(vendor_tests_pattern) |
| 234 | for test_file_path in test_file_paths: |
| 235 | known_tests.Parse(test_file_path) |
| 236 | if os.path.isfile(self._options.user_tests_file): |
| 237 | known_tests.Parse(self._options.user_tests_file) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 238 | return known_tests |
| 239 | except errors.ParseError: |
| 240 | raise errors.AbortError |
| 241 | |
| 242 | def _DumpTests(self): |
| 243 | """Prints out set of defined tests.""" |
Brett Chabot | be659c0 | 2009-09-21 17:48:26 -0700 | [diff] [blame] | 244 | print "The following tests are currently defined:\n" |
| 245 | print "%-25s %-40s %s" % ("name", "build path", "description") |
| 246 | print "-" * 80 |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 247 | for test in self._known_tests: |
Brett Chabot | be659c0 | 2009-09-21 17:48:26 -0700 | [diff] [blame] | 248 | print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(), |
| 249 | test.GetDescription()) |
| 250 | print "\nSee %s for more information" % self._TEST_FILE_NAME |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 251 | |
| 252 | def _DoBuild(self): |
| 253 | logger.SilentLog("Building tests...") |
Brett Chabot | 8dc9eb8 | 2010-04-15 15:43:04 -0700 | [diff] [blame] | 254 | tests = self._GetTestsToRun() |
Santos Cordon | 4e0ad8f | 2015-05-21 12:25:05 -0700 | [diff] [blame] | 255 | |
| 256 | # Build and install tests that do not get granted permissions |
| 257 | self._DoPermissionAwareBuild(tests, False) |
| 258 | |
| 259 | # Build and install tests that require granted permissions |
| 260 | self._DoPermissionAwareBuild(tests, True) |
| 261 | |
| 262 | def _DoPermissionAwareBuild(self, tests, test_requires_permissions): |
Brett Chabot | b1eb5d2 | 2010-06-15 11:18:42 -0700 | [diff] [blame] | 263 | # turn off dalvik verifier if necessary |
Brett Chabot | 8b538bd | 2014-04-22 11:46:00 -0700 | [diff] [blame] | 264 | # TODO: skip turning off verifier for now, since it puts device in bad |
| 265 | # state b/14088982 |
| 266 | #self._TurnOffVerifier(tests) |
Santos Cordon | 4e0ad8f | 2015-05-21 12:25:05 -0700 | [diff] [blame] | 267 | self._DoFullBuild(tests, test_requires_permissions) |
Brett Chabot | 8dc9eb8 | 2010-04-15 15:43:04 -0700 | [diff] [blame] | 268 | |
Brett Chabot | 8ac5118 | 2012-09-19 07:35:35 -0700 | [diff] [blame] | 269 | target_tree = make_tree.MakeTree() |
Brian Muramatsu | 95cbc9e | 2012-01-10 12:06:12 -0800 | [diff] [blame] | 270 | |
| 271 | extra_args_set = [] |
Brett Chabot | 2477b38 | 2009-09-23 18:05:28 -0700 | [diff] [blame] | 272 | for test_suite in tests: |
Santos Cordon | 4e0ad8f | 2015-05-21 12:25:05 -0700 | [diff] [blame] | 273 | if test_suite.IsGrantedPermissions() == test_requires_permissions: |
| 274 | self._AddBuildTarget(test_suite, target_tree, extra_args_set) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 275 | |
Brett Chabot | ccae47d | 2010-06-14 15:19:25 -0700 | [diff] [blame] | 276 | if not self._options.preview: |
| 277 | self._adb.EnableAdbRoot() |
| 278 | else: |
| 279 | logger.Log("adb root") |
Brett Chabot | 8ac5118 | 2012-09-19 07:35:35 -0700 | [diff] [blame] | 280 | |
| 281 | if not target_tree.IsEmpty(): |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 282 | if self._options.coverage: |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 283 | coverage.EnableCoverageBuild() |
Brett Chabot | 8ac5118 | 2012-09-19 07:35:35 -0700 | [diff] [blame] | 284 | target_tree.AddPath("external/emma") |
Brett Chabot | 2477b38 | 2009-09-23 18:05:28 -0700 | [diff] [blame] | 285 | |
Brett Chabot | 8ac5118 | 2012-09-19 07:35:35 -0700 | [diff] [blame] | 286 | target_list = target_tree.GetPrunedMakeList() |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 287 | target_dir_list = [re.sub(r'Android[.]mk$', r'', i) for i in target_list] |
Brett Chabot | 8ac5118 | 2012-09-19 07:35:35 -0700 | [diff] [blame] | 288 | target_build_string = " ".join(target_list) |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 289 | target_dir_build_string = " ".join(target_dir_list) |
Brian Muramatsu | 95cbc9e | 2012-01-10 12:06:12 -0800 | [diff] [blame] | 290 | extra_args_string = " ".join(extra_args_set) |
Brett Chabot | b1eb5d2 | 2010-06-15 11:18:42 -0700 | [diff] [blame] | 291 | |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 292 | # mmm cannot be used from python, so perform a similar operation using |
Brett Chabot | 2b6643b | 2009-04-07 18:35:27 -0700 | [diff] [blame] | 293 | # ONE_SHOT_MAKEFILE |
Brett Chabot | 5f5928c | 2013-08-20 17:06:03 -0700 | [diff] [blame] | 294 | cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" GET-INSTALL-PATH all_modules %s' % ( |
Brett Chabot | 2477b38 | 2009-09-23 18:05:28 -0700 | [diff] [blame] | 295 | target_build_string, self._options.make_jobs, self._root_path, |
| 296 | extra_args_string) |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 297 | # mmma equivalent, used when regular mmm fails |
| 298 | alt_cmd = 'make -j%s -C "%s" -f build/core/main.mk %s all_modules BUILD_MODULES_IN_PATHS="%s"' % ( |
| 299 | self._options.make_jobs, self._root_path, extra_args_string, target_dir_build_string) |
| 300 | |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 301 | logger.Log(cmd) |
Brett Chabot | 7454171 | 2012-08-31 18:39:00 -0700 | [diff] [blame] | 302 | if not self._options.preview: |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 303 | run_command.SetAbortOnError() |
| 304 | try: |
| 305 | output = run_command.RunCommand(cmd, return_output=True, timeout_time=600) |
| 306 | ## Chances are this failed because it didn't build the dependencies |
| 307 | except errors.AbortError: |
| 308 | logger.Log("make failed. Trying to rebuild all dependencies.") |
| 309 | logger.Log("mmma -j%s %s" %(self._options.make_jobs, target_dir_build_string)) |
| 310 | # Try again with mma equivalent, which will build the dependencies |
| 311 | run_command.RunCommand(alt_cmd, return_output=False, timeout_time=600) |
| 312 | # Run mmm again to get the install paths only |
| 313 | output = run_command.RunCommand(cmd, return_output=True, timeout_time=600) |
| 314 | run_command.SetAbortOnError(False) |
Brett Chabot | 5f5928c | 2013-08-20 17:06:03 -0700 | [diff] [blame] | 315 | logger.SilentLog(output) |
Santos Cordon | 4e0ad8f | 2015-05-21 12:25:05 -0700 | [diff] [blame] | 316 | self._DoInstall(output, test_requires_permissions) |
Niko Catania | a6dc2ab | 2009-04-03 14:12:46 -0700 | [diff] [blame] | 317 | |
Santos Cordon | 4e0ad8f | 2015-05-21 12:25:05 -0700 | [diff] [blame] | 318 | def _DoInstall(self, make_output, test_requires_permissions): |
Brett Chabot | 7454171 | 2012-08-31 18:39:00 -0700 | [diff] [blame] | 319 | """Install artifacts from build onto device. |
| 320 | |
| 321 | Looks for 'install:' text from make output to find artifacts to install. |
| 322 | |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 323 | Files with the .apk extension get 'adb install'ed, all other files |
| 324 | get 'adb push'ed onto the device. |
| 325 | |
Brett Chabot | 7454171 | 2012-08-31 18:39:00 -0700 | [diff] [blame] | 326 | Args: |
| 327 | make_output: stdout from make command |
| 328 | """ |
| 329 | for line in make_output.split("\n"): |
| 330 | m = self._RE_MAKE_INSTALL.match(line) |
| 331 | if m: |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 332 | # strip the 'INSTALL: <name>' from the left hand side |
| 333 | # the remaining string is a space-separated list of build-generated files |
| 334 | install_paths = m.group(2) |
| 335 | for install_path in re.split(r'\s+', install_paths): |
| 336 | if install_path.endswith(".apk"): |
| 337 | abs_install_path = os.path.join(self._root_path, install_path) |
Santos Cordon | 4e0ad8f | 2015-05-21 12:25:05 -0700 | [diff] [blame] | 338 | extra_flags = "" |
| 339 | if test_requires_permissions and not self._options.skip_permissions: |
| 340 | extra_flags = "-g" |
| 341 | logger.Log("adb install -r %s %s" % (extra_flags, abs_install_path)) |
| 342 | logger.Log(self._adb.Install(abs_install_path, extra_flags)) |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 343 | else: |
| 344 | self._PushInstallFileToDevice(install_path) |
Brett Chabot | 7454171 | 2012-08-31 18:39:00 -0700 | [diff] [blame] | 345 | |
| 346 | def _PushInstallFileToDevice(self, install_path): |
JP Abgrall | f38107c | 2013-07-11 17:39:16 -0700 | [diff] [blame] | 347 | m = self._re_make_install_path.match(install_path) |
Brett Chabot | 7454171 | 2012-08-31 18:39:00 -0700 | [diff] [blame] | 348 | if m: |
| 349 | remote_path = m.group(1) |
Brett Chabot | e607d3a | 2013-05-16 23:00:43 -0700 | [diff] [blame] | 350 | remote_dir = os.path.dirname(remote_path) |
| 351 | logger.Log("adb shell mkdir -p %s" % remote_dir) |
| 352 | self._adb.SendShellCommand("mkdir -p %s" % remote_dir) |
Brett Chabot | 7454171 | 2012-08-31 18:39:00 -0700 | [diff] [blame] | 353 | abs_install_path = os.path.join(self._root_path, install_path) |
Brett Chabot | 81c475e | 2012-09-11 12:57:31 -0700 | [diff] [blame] | 354 | logger.Log("adb push %s %s" % (abs_install_path, remote_path)) |
Brett Chabot | 7454171 | 2012-08-31 18:39:00 -0700 | [diff] [blame] | 355 | self._adb.Push(abs_install_path, remote_path) |
| 356 | else: |
| 357 | logger.Log("Error: Failed to recognize path of file to install %s" % install_path) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 358 | |
Santos Cordon | 4e0ad8f | 2015-05-21 12:25:05 -0700 | [diff] [blame] | 359 | def _DoFullBuild(self, tests, test_requires_permissions): |
Brett Chabot | 8dc9eb8 | 2010-04-15 15:43:04 -0700 | [diff] [blame] | 360 | """If necessary, run a full 'make' command for the tests that need it.""" |
| 361 | extra_args_set = Set() |
| 362 | |
Brett Chabot | 8dc9eb8 | 2010-04-15 15:43:04 -0700 | [diff] [blame] | 363 | for test in tests: |
Santos Cordon | 4e0ad8f | 2015-05-21 12:25:05 -0700 | [diff] [blame] | 364 | if test.IsFullMake() and test.IsGrantedPermissions() == test_requires_permissions: |
Brett Chabot | 8dc9eb8 | 2010-04-15 15:43:04 -0700 | [diff] [blame] | 365 | if test.GetExtraBuildArgs(): |
| 366 | # extra args contains the args to pass to 'make' |
| 367 | extra_args_set.add(test.GetExtraBuildArgs()) |
| 368 | else: |
| 369 | logger.Log("Warning: test %s needs a full build but does not specify" |
| 370 | " extra_build_args" % test.GetName()) |
| 371 | |
| 372 | # check if there is actually any tests that required a full build |
| 373 | if extra_args_set: |
| 374 | cmd = ('make -j%s %s' % (self._options.make_jobs, |
| 375 | ' '.join(list(extra_args_set)))) |
| 376 | logger.Log(cmd) |
| 377 | if not self._options.preview: |
| 378 | old_dir = os.getcwd() |
| 379 | os.chdir(self._root_path) |
Brett Chabot | b0b8c78 | 2012-09-19 08:32:56 -0700 | [diff] [blame] | 380 | output = run_command.RunCommand(cmd, return_output=True) |
Brett Chabot | 5f5928c | 2013-08-20 17:06:03 -0700 | [diff] [blame] | 381 | logger.SilentLog(output) |
Brett Chabot | 8dc9eb8 | 2010-04-15 15:43:04 -0700 | [diff] [blame] | 382 | os.chdir(old_dir) |
Santos Cordon | 4e0ad8f | 2015-05-21 12:25:05 -0700 | [diff] [blame] | 383 | self._DoInstall(output, test_requires_permissions) |
Brett Chabot | 8dc9eb8 | 2010-04-15 15:43:04 -0700 | [diff] [blame] | 384 | |
Brett Chabot | 8ac5118 | 2012-09-19 07:35:35 -0700 | [diff] [blame] | 385 | def _AddBuildTarget(self, test_suite, target_tree, extra_args_set): |
Brett Chabot | 8dc9eb8 | 2010-04-15 15:43:04 -0700 | [diff] [blame] | 386 | if not test_suite.IsFullMake(): |
| 387 | build_dir = test_suite.GetBuildPath() |
Brett Chabot | 8ac5118 | 2012-09-19 07:35:35 -0700 | [diff] [blame] | 388 | if self._AddBuildTargetPath(build_dir, target_tree): |
Brian Muramatsu | 95cbc9e | 2012-01-10 12:06:12 -0800 | [diff] [blame] | 389 | extra_args_set.append(test_suite.GetExtraBuildArgs()) |
Brett Chabot | 8dc9eb8 | 2010-04-15 15:43:04 -0700 | [diff] [blame] | 390 | for path in test_suite.GetBuildDependencies(self._options): |
Brett Chabot | 8ac5118 | 2012-09-19 07:35:35 -0700 | [diff] [blame] | 391 | self._AddBuildTargetPath(path, target_tree) |
Brett Chabot | 2b6643b | 2009-04-07 18:35:27 -0700 | [diff] [blame] | 392 | |
Brett Chabot | 8ac5118 | 2012-09-19 07:35:35 -0700 | [diff] [blame] | 393 | def _AddBuildTargetPath(self, build_dir, target_tree): |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 394 | if build_dir is not None: |
Brett Chabot | 8ac5118 | 2012-09-19 07:35:35 -0700 | [diff] [blame] | 395 | target_tree.AddPath(build_dir) |
| 396 | return True |
Brett Chabot | 2b6643b | 2009-04-07 18:35:27 -0700 | [diff] [blame] | 397 | return False |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 398 | |
| 399 | def _GetTestsToRun(self): |
| 400 | """Get a list of TestSuite objects to run, based on command line args.""" |
Brett Chabot | 59b4778 | 2009-10-21 17:23:01 -0700 | [diff] [blame] | 401 | if self._tests_to_run: |
| 402 | return self._tests_to_run |
| 403 | |
| 404 | self._tests_to_run = [] |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 405 | if self._options.all_tests: |
Brett Chabot | 59b4778 | 2009-10-21 17:23:01 -0700 | [diff] [blame] | 406 | self._tests_to_run = self._known_tests.GetTests() |
Brett Chabot | 49b7711 | 2009-06-02 11:46:04 -0700 | [diff] [blame] | 407 | elif self._options.continuous_tests: |
Brett Chabot | 59b4778 | 2009-10-21 17:23:01 -0700 | [diff] [blame] | 408 | self._tests_to_run = self._known_tests.GetContinuousTests() |
Brett Chabot | 4a5d9f1 | 2010-02-18 20:01:11 -0800 | [diff] [blame] | 409 | elif self._options.suite: |
| 410 | self._tests_to_run = \ |
| 411 | self._known_tests.GetTestsInSuite(self._options.suite) |
Brett Chabot | 59b4778 | 2009-10-21 17:23:01 -0700 | [diff] [blame] | 412 | elif self._options.test_path: |
| 413 | walker = test_walker.TestWalker() |
| 414 | self._tests_to_run = walker.FindTests(self._options.test_path) |
| 415 | |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 416 | for name in self._test_args: |
| 417 | test = self._known_tests.GetTest(name) |
| 418 | if test is None: |
| 419 | logger.Log("Error: Could not find test %s" % name) |
| 420 | self._DumpTests() |
| 421 | raise errors.AbortError |
Brett Chabot | 59b4778 | 2009-10-21 17:23:01 -0700 | [diff] [blame] | 422 | self._tests_to_run.append(test) |
| 423 | return self._tests_to_run |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 424 | |
Brett Chabot | 2477b38 | 2009-09-23 18:05:28 -0700 | [diff] [blame] | 425 | def _IsCtsTests(self, test_list): |
| 426 | """Check if any cts tests are included in given list of tests to run.""" |
| 427 | for test in test_list: |
Brett Chabot | 4a5d9f1 | 2010-02-18 20:01:11 -0800 | [diff] [blame] | 428 | if test.GetSuite() == 'cts': |
Brett Chabot | 2477b38 | 2009-09-23 18:05:28 -0700 | [diff] [blame] | 429 | return True |
| 430 | return False |
| 431 | |
Brett Chabot | ccae47d | 2010-06-14 15:19:25 -0700 | [diff] [blame] | 432 | def _TurnOffVerifier(self, test_list): |
| 433 | """Turn off the dalvik verifier if needed by given tests. |
| 434 | |
| 435 | If one or more tests needs dalvik verifier off, and it is not already off, |
| 436 | turns off verifier and reboots device to allow change to take effect. |
| 437 | """ |
Brett Chabot | 7454171 | 2012-08-31 18:39:00 -0700 | [diff] [blame] | 438 | # hack to check if these are frameworks/base tests. If so, turn off verifier |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 439 | # to allow framework tests to access private/protected/package-private framework api |
Brett Chabot | ccae47d | 2010-06-14 15:19:25 -0700 | [diff] [blame] | 440 | framework_test = False |
| 441 | for test in test_list: |
| 442 | if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]): |
| 443 | framework_test = True |
| 444 | if framework_test: |
| 445 | # check if verifier is off already - to avoid the reboot if not |
| 446 | # necessary |
| 447 | output = self._adb.SendShellCommand("cat /data/local.prop") |
| 448 | if not self._DALVIK_VERIFIER_OFF_PROP in output: |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 449 | |
| 450 | # Read the existing dalvik verifier flags. |
| 451 | old_prop_value = self._adb.SendShellCommand("getprop %s" \ |
| 452 | %(self._DALVIK_VERIFIER_PROP)) |
| 453 | old_prop_value = old_prop_value.strip() if old_prop_value else "" |
| 454 | |
| 455 | # Append our verifier flags to existing flags |
| 456 | new_prop_value = "%s %s" %(self._DALVIK_VERIFIER_OFF_VALUE, old_prop_value) |
| 457 | |
| 458 | # Update property now, as /data/local.prop is not read until reboot |
| 459 | logger.Log("adb shell setprop %s '%s'" \ |
| 460 | %(self._DALVIK_VERIFIER_PROP, new_prop_value)) |
| 461 | if not self._options.preview: |
| 462 | self._adb.SendShellCommand("setprop %s '%s'" \ |
| 463 | %(self._DALVIK_VERIFIER_PROP, new_prop_value)) |
| 464 | |
| 465 | # Write prop to /data/local.prop |
| 466 | # Every time device is booted, it will pick up this value |
| 467 | new_prop_assignment = "%s = %s" %(self._DALVIK_VERIFIER_PROP, new_prop_value) |
Brett Chabot | ccae47d | 2010-06-14 15:19:25 -0700 | [diff] [blame] | 468 | if self._options.preview: |
| 469 | logger.Log("adb shell \"echo %s >> /data/local.prop\"" |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 470 | % new_prop_assignment) |
Brett Chabot | 25dfd79 | 2012-02-16 15:51:43 -0800 | [diff] [blame] | 471 | logger.Log("adb shell chmod 644 /data/local.prop") |
Brett Chabot | ccae47d | 2010-06-14 15:19:25 -0700 | [diff] [blame] | 472 | else: |
| 473 | logger.Log("Turning off dalvik verifier and rebooting") |
| 474 | self._adb.SendShellCommand("\"echo %s >> /data/local.prop\"" |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 475 | % new_prop_assignment) |
Brett Chabot | 25dfd79 | 2012-02-16 15:51:43 -0800 | [diff] [blame] | 476 | |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 477 | # Reset runtime so that dalvik picks up new verifier flags from prop |
| 478 | self._ChmodRuntimeReset() |
Brett Chabot | 25dfd79 | 2012-02-16 15:51:43 -0800 | [diff] [blame] | 479 | elif not self._options.preview: |
| 480 | # check the permissions on the file |
| 481 | permout = self._adb.SendShellCommand("ls -l /data/local.prop") |
| 482 | if not "-rw-r--r--" in permout: |
| 483 | logger.Log("Fixing permissions on /data/local.prop and rebooting") |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 484 | self._ChmodRuntimeReset() |
Brett Chabot | 25dfd79 | 2012-02-16 15:51:43 -0800 | [diff] [blame] | 485 | |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 486 | def _ChmodRuntimeReset(self): |
| 487 | """Perform a chmod of /data/local.prop and reset the runtime. |
Brett Chabot | 25dfd79 | 2012-02-16 15:51:43 -0800 | [diff] [blame] | 488 | """ |
Igor Murashkin | a0afc8c | 2014-01-22 16:22:50 -0800 | [diff] [blame] | 489 | logger.Log("adb shell chmod 644 /data/local.prop ## u+w,a+r") |
| 490 | if not self._options.preview: |
| 491 | self._adb.SendShellCommand("chmod 644 /data/local.prop") |
| 492 | |
| 493 | self._adb.RuntimeReset(preview_only=self._options.preview) |
| 494 | |
| 495 | if not self._options.preview: |
| 496 | self._adb.EnableAdbRoot() |
Brett Chabot | 25dfd79 | 2012-02-16 15:51:43 -0800 | [diff] [blame] | 497 | |
Brett Chabot | ccae47d | 2010-06-14 15:19:25 -0700 | [diff] [blame] | 498 | |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 499 | def RunTests(self): |
| 500 | """Main entry method - executes the tests according to command line args.""" |
| 501 | try: |
| 502 | run_command.SetAbortOnError() |
| 503 | self._ProcessOptions() |
| 504 | if self._options.only_list_tests: |
| 505 | self._DumpTests() |
| 506 | return |
| 507 | |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 508 | if not self._options.skip_build: |
| 509 | self._DoBuild() |
| 510 | |
Igor Murashkin | 8d70353 | 2014-01-23 16:13:29 -0800 | [diff] [blame] | 511 | if self._options.build_install_only: |
| 512 | logger.Log("Skipping test execution (due to --build-install-only flag)") |
| 513 | return |
| 514 | |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 515 | for test_suite in self._GetTestsToRun(): |
Brett Chabot | 920e9fe | 2010-01-21 17:30:47 -0800 | [diff] [blame] | 516 | try: |
| 517 | test_suite.Run(self._options, self._adb) |
| 518 | except errors.WaitForResponseTimedOutError: |
| 519 | logger.Log("Timed out waiting for response") |
Brett Chabot | 764d3fa | 2009-06-25 17:57:31 -0700 | [diff] [blame] | 520 | |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 521 | except KeyboardInterrupt: |
| 522 | logger.Log("Exiting...") |
Brett Chabot | 3ae5f8a | 2009-06-28 12:00:47 -0700 | [diff] [blame] | 523 | except errors.AbortError, error: |
| 524 | logger.Log(error.msg) |
The Android Open Source Project | 6ffae01 | 2009-03-18 17:39:43 -0700 | [diff] [blame] | 525 | logger.SilentLog("Exiting due to AbortError...") |
| 526 | except errors.WaitForResponseTimedOutError: |
| 527 | logger.Log("Timed out waiting for response") |
| 528 | |
| 529 | |
| 530 | def RunTests(): |
| 531 | runner = TestRunner() |
| 532 | runner.RunTests() |
| 533 | |
| 534 | if __name__ == "__main__": |
| 535 | RunTests() |