Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # |
| 3 | # Copyright 2017, The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 17 | """ |
| 18 | Command line utility for running Android tests through TradeFederation. |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 19 | |
| 20 | atest helps automate the flow of building test modules across the Android |
| 21 | code base and executing the tests via the TradeFederation test harness. |
| 22 | |
| 23 | atest is designed to support any test types that can be ran by TradeFederation. |
| 24 | """ |
| 25 | |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 26 | import logging |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 27 | import os |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 28 | import subprocess |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 29 | import sys |
mikehoran | 95091b2 | 2017-10-31 15:55:26 -0700 | [diff] [blame] | 30 | import tempfile |
| 31 | import time |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 32 | |
Simran Basi | cf2189b | 2017-11-06 23:40:24 -0800 | [diff] [blame] | 33 | import atest_utils |
mikehoran | 63d61b4 | 2017-07-28 15:28:50 -0700 | [diff] [blame] | 34 | import cli_translator |
Kevin Cheng | 7edb0b9 | 2017-12-14 15:00:25 -0800 | [diff] [blame] | 35 | # pylint: disable=import-error |
| 36 | import constants |
| 37 | import test_runner_handler |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame^] | 38 | from test_runners import regression_test_runner |
Simran Basi | cf2189b | 2017-11-06 23:40:24 -0800 | [diff] [blame] | 39 | |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 40 | EXPECTED_VARS = frozenset([ |
Simran Basi | cf2189b | 2017-11-06 23:40:24 -0800 | [diff] [blame] | 41 | atest_utils.ANDROID_BUILD_TOP, |
mikehoran | 43ed32d | 2017-08-18 17:13:36 -0700 | [diff] [blame] | 42 | 'ANDROID_TARGET_OUT_TESTCASES', |
| 43 | 'OUT']) |
mikehoran | c80dc53 | 2017-11-14 14:30:06 -0800 | [diff] [blame] | 44 | BUILD_STEP = 'build' |
mikehoran | c327dca | 2017-11-27 16:24:22 -0800 | [diff] [blame] | 45 | INSTALL_STEP = 'install' |
mikehoran | c80dc53 | 2017-11-14 14:30:06 -0800 | [diff] [blame] | 46 | TEST_STEP = 'test' |
mikehoran | c327dca | 2017-11-27 16:24:22 -0800 | [diff] [blame] | 47 | ALL_STEPS = [BUILD_STEP, INSTALL_STEP, TEST_STEP] |
mikehoran | 95091b2 | 2017-10-31 15:55:26 -0700 | [diff] [blame] | 48 | TEST_RUN_DIR_PREFIX = 'atest_run_%s_' |
mikehoran | d229b1b | 2017-12-01 15:23:58 -0800 | [diff] [blame] | 49 | HELP_DESC = '''Build, install and run Android tests locally.''' |
Simran Basi | 05384b8 | 2018-01-03 16:19:30 -0800 | [diff] [blame] | 50 | REBUILD_MODULE_INFO_FLAG = '--rebuild-module-info' |
mikehoran | c80dc53 | 2017-11-14 14:30:06 -0800 | [diff] [blame] | 51 | |
mikehoran | d229b1b | 2017-12-01 15:23:58 -0800 | [diff] [blame] | 52 | EPILOG_TEXT = ''' |
| 53 | |
| 54 | |
| 55 | - - - - - - - - - |
| 56 | IDENTIFYING TESTS |
| 57 | - - - - - - - - - |
| 58 | |
| 59 | The positional argument <tests> should be a reference to one or more of the |
| 60 | tests you'd like to run. Multiple tests can be run in one command by |
| 61 | separating test references with spaces. |
| 62 | |
| 63 | Usage Template: atest <reference_to_test_1> <reference_to_test_2> |
| 64 | |
| 65 | A <reference_to_test> can be satisfied by the test's MODULE NAME, |
mikehoran | 32861ae | 2017-12-15 11:53:17 -0800 | [diff] [blame] | 66 | MODULE:CLASS, CLASS NAME, TF INTEGRATION TEST or FILE PATH. Explanations |
mikehoran | d229b1b | 2017-12-01 15:23:58 -0800 | [diff] [blame] | 67 | and examples of each follow. |
| 68 | |
| 69 | |
| 70 | < MODULE NAME > |
| 71 | |
| 72 | Identifying a test by its module name will run the entire module. Input |
| 73 | the name as it appears in the LOCAL_MODULE or LOCAL_PACKAGE_NAME |
| 74 | variables in that test's Android.mk or Android.bp file. |
| 75 | |
| 76 | Note: Use < TF INTEGRATION TEST > to run non-module tests integrated |
| 77 | directly into TradeFed. |
| 78 | |
| 79 | Examples: |
| 80 | atest FrameworksServicesTests |
| 81 | atest CtsJankDeviceTestCases |
| 82 | |
| 83 | |
mikehoran | d229b1b | 2017-12-01 15:23:58 -0800 | [diff] [blame] | 84 | < MODULE:CLASS > |
| 85 | |
| 86 | Identifying a test by its class name will run just the tests in that |
| 87 | class and not the whole module. MODULE:CLASS is the preferred way to run |
| 88 | a single class. MODULE is the same as described above. CLASS is the |
| 89 | name of the test class in the .java file. It can either be the fully |
| 90 | qualified class name or just the basic name. |
| 91 | |
| 92 | Examples: |
| 93 | atest PtsBatteryTestCases:BatteryTest |
| 94 | atest PtsBatteryTestCases:com.google.android.battery.pts.BatteryTest |
| 95 | atest CtsJankDeviceTestCases:CtsDeviceJankUi |
| 96 | |
| 97 | |
| 98 | < CLASS NAME > |
| 99 | |
| 100 | A single class can also be run by referencing the class name without |
| 101 | the module name. However, this will take more time than the equivalent |
| 102 | MODULE:CLASS reference, so we suggest using a MODULE:CLASS reference |
| 103 | whenever possible. |
| 104 | |
| 105 | Examples: |
| 106 | atest ScreenDecorWindowTests |
| 107 | atest com.google.android.battery.pts.BatteryTest |
| 108 | atest CtsDeviceJankUi |
| 109 | |
| 110 | |
mikehoran | 32861ae | 2017-12-15 11:53:17 -0800 | [diff] [blame] | 111 | < TF INTEGRATION TEST > |
| 112 | |
| 113 | To run tests that are integrated directly into TradeFed (non-modules), |
| 114 | input the name as it appears in the output of the "tradefed.sh list |
| 115 | configs" cmd. |
| 116 | |
| 117 | Examples: |
| 118 | atest example/reboot |
| 119 | atest native-benchmark |
| 120 | |
| 121 | |
mikehoran | d229b1b | 2017-12-01 15:23:58 -0800 | [diff] [blame] | 122 | < FILE PATH > |
| 123 | |
| 124 | Both module-based tests and integration-based tests can be run by |
| 125 | inputting the path to their test file or dir as appropriate. A single |
| 126 | class can also be run by inputting the path to the class's java file. |
| 127 | Both relative and absolute paths are supported. |
| 128 | |
| 129 | Example - run module from android repo root: |
| 130 | atest cts/tests/jank/jank |
| 131 | |
| 132 | Example - same module but from <repo root>/cts/tests/jank: |
| 133 | atest . |
| 134 | |
| 135 | Example - run just class from android repo root: |
| 136 | atest cts/tests/jank/src/android/jank/cts/ui/CtsDeviceJankUi.java |
| 137 | |
| 138 | Example - run tf integration test from android repo root: |
| 139 | atest tools/tradefederation/contrib/res/config/example/reboot.xml |
| 140 | |
| 141 | |
| 142 | - - - - - - - - - - - - - - - - - - - - - - - - - - |
| 143 | SPECIFYING INDIVIDUAL STEPS: BUILD, INSTALL OR RUN |
| 144 | - - - - - - - - - - - - - - - - - - - - - - - - - - |
| 145 | |
| 146 | The -b, -i and -t options allow you to specify which steps you want to run. |
| 147 | If none of those options are given, then all steps are run. If any of these |
| 148 | options are provided then only the listed steps are run. |
| 149 | |
| 150 | Note: -i alone is not currently support and can only be included with -t. |
| 151 | Both -b and -t can be run alone. |
| 152 | |
| 153 | Examples: |
| 154 | atest -b <test> (just build targets) |
| 155 | atest -bt <test> (build targets, run tests, but skip installing apk) |
| 156 | atest -t <test> (just run test, skip build/install) |
| 157 | atest -it <test> (install and run tests, skip building) |
| 158 | |
| 159 | |
| 160 | - - - - - - - - - - - - - |
| 161 | RUNNING SPECIFIC METHODS |
| 162 | - - - - - - - - - - - - - |
| 163 | |
| 164 | It is possible to run only specific methods within a test class. To run |
| 165 | only specific methods, identify the class in any of the ways supported |
| 166 | for identifying a class (MODULE:CLASS, FILE PATH, etc) and then append the |
| 167 | name of the method or method using the following template: |
| 168 | |
| 169 | <reference_to_class>#<method1>,<method2>,<method3>... |
| 170 | |
| 171 | Examples: |
| 172 | FrameworksServicesTests:ScreenDecorWindowTests#testFlagChange,testRemoval |
| 173 | com.google.android.battery.pts.BatteryTest#testDischarge |
| 174 | |
| 175 | |
| 176 | - - - - - - - - - - - - - |
| 177 | RUNNING MULTIPLE CLASSES |
| 178 | - - - - - - - - - - - - - |
| 179 | |
| 180 | To run multiple classes, deliminate them with spaces just like you would |
| 181 | if running multiple tests. Atest will automatically build and run |
| 182 | multiple classes in the most efficient way possible. |
| 183 | |
| 184 | |
| 185 | Example - two classes in same module: |
| 186 | atest FrameworksServicesTests:ScreenDecorWindowTests FrameworksServicesTest:DimmerTests |
| 187 | |
| 188 | Example - two classes, different modules: |
| 189 | atest FrameworksServicesTests:ScreenDecorWindowTests CtsJankDeviceTestCases:CtsDeviceJankUi |
| 190 | |
Mike Ma | 150a61d | 2017-12-15 10:53:35 -0800 | [diff] [blame] | 191 | |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame^] | 192 | - - - - - - - - - - - |
| 193 | REGRESSION DETECTION |
| 194 | - - - - - - - - - - - |
| 195 | |
| 196 | Generate pre-patch or post-patch metrics without running regression detection: |
| 197 | |
| 198 | Example: |
| 199 | atest <test> --generate-baseline <optional iter> |
| 200 | atest <test> --generate-new-metrics <optional iter> |
| 201 | |
| 202 | Local regression detection can be run in three options: |
| 203 | |
| 204 | 1) Provide a folder containing baseline (pre-patch) metrics (generated previously). Atest will |
| 205 | run the tests n (default 5) iterations, generate a new set of post-patch metrics, and |
| 206 | compare those against existing metrics. |
| 207 | |
| 208 | Example: |
| 209 | atest <test> --detect-regression </path/to/baseline> --generate-new-metrics <optional iter> |
| 210 | |
| 211 | 2) Provide a folder containing post-patch metrics (generated previously). Atest will run the |
| 212 | tests n (default 5) iterations, generate a new set of pre-patch metrics, and compare those |
| 213 | against those provided. Note: the developer needs to revert the device/tests to pre-patch |
| 214 | state to generate baseline metrics. |
| 215 | |
| 216 | Example: |
| 217 | atest <test> --detect-regression </path/to/new> --generate-baseline <optional iter> |
| 218 | |
| 219 | 3) Provide 2 folders containing both pre-patch and post-patch metrics. Atest will run no tests |
| 220 | but the regression detection algorithm. |
| 221 | |
| 222 | Example: |
| 223 | atest --detect-regression </path/to/baseline> </path/to/new> |
| 224 | |
| 225 | |
mikehoran | c80dc53 | 2017-11-14 14:30:06 -0800 | [diff] [blame] | 226 | ''' |
| 227 | |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 228 | |
mikehoran | 63d61b4 | 2017-07-28 15:28:50 -0700 | [diff] [blame] | 229 | def _parse_args(argv): |
| 230 | """Parse command line arguments. |
| 231 | |
| 232 | Args: |
| 233 | argv: A list of arguments. |
| 234 | |
| 235 | Returns: |
| 236 | An argspace.Namespace class instance holding parsed args. |
| 237 | """ |
| 238 | import argparse |
| 239 | parser = argparse.ArgumentParser( |
mikehoran | c80dc53 | 2017-11-14 14:30:06 -0800 | [diff] [blame] | 240 | description=HELP_DESC, |
mikehoran | d229b1b | 2017-12-01 15:23:58 -0800 | [diff] [blame] | 241 | epilog=EPILOG_TEXT, |
mikehoran | 7592624 | 2017-09-07 11:01:35 -0700 | [diff] [blame] | 242 | formatter_class=argparse.RawTextHelpFormatter) |
mikehoran | d229b1b | 2017-12-01 15:23:58 -0800 | [diff] [blame] | 243 | parser.add_argument('tests', nargs='*', help='Tests to build and/or run.') |
mikehoran | c80dc53 | 2017-11-14 14:30:06 -0800 | [diff] [blame] | 244 | parser.add_argument('-b', '--build', action='append_const', dest='steps', |
| 245 | const=BUILD_STEP, help='Run a build.') |
mikehoran | c327dca | 2017-11-27 16:24:22 -0800 | [diff] [blame] | 246 | parser.add_argument('-i', '--install', action='append_const', dest='steps', |
| 247 | const=INSTALL_STEP, help='Install an APK.') |
mikehoran | c80dc53 | 2017-11-14 14:30:06 -0800 | [diff] [blame] | 248 | parser.add_argument('-t', '--test', action='append_const', dest='steps', |
| 249 | const=TEST_STEP, help='Run the tests.') |
Simran Basi | 05384b8 | 2018-01-03 16:19:30 -0800 | [diff] [blame] | 250 | parser.add_argument('-m', REBUILD_MODULE_INFO_FLAG, action='store_true', |
| 251 | help='Forces a rebuild of the module-info.json file. ' |
| 252 | 'This may be necessary following a repo sync or ' |
| 253 | 'when writing a new test.') |
Simran Basi | 67ba20b | 2017-11-01 19:01:48 -0700 | [diff] [blame] | 254 | parser.add_argument('-w', '--wait-for-debugger', action='store_true', |
| 255 | help='Only for instrumentation tests. Waits for ' |
| 256 | 'debugger prior to execution.') |
mikehoran | c80dc53 | 2017-11-14 14:30:06 -0800 | [diff] [blame] | 257 | parser.add_argument('-v', '--verbose', action='store_true', |
| 258 | help='Display DEBUG level logging.') |
Mike Ma | 150a61d | 2017-12-15 10:53:35 -0800 | [diff] [blame] | 259 | parser.add_argument('--generate-baseline', nargs='?', type=int, const=5, default=0, |
| 260 | help='Generate baseline metrics, run 5 iterations by default. ' |
| 261 | 'Provide an int argument to specify # iterations.') |
| 262 | parser.add_argument('--generate-new-metrics', nargs='?', type=int, const=5, default=0, |
| 263 | help='Generate new metrics, run 5 iterations by default. ' |
| 264 | 'Provide an int argument to specify # iterations.') |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame^] | 265 | parser.add_argument('--detect-regression', nargs='*', |
| 266 | help='Run regression detection algorithm. Supply ' |
| 267 | 'path to baseline and/or new metrics folders.') |
mikehoran | 63d61b4 | 2017-07-28 15:28:50 -0700 | [diff] [blame] | 268 | return parser.parse_args(argv) |
| 269 | |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 270 | |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 271 | def _configure_logging(verbose): |
| 272 | """Configure the logger. |
| 273 | |
| 274 | Args: |
| 275 | verbose: A boolean. If true display DEBUG level logs. |
| 276 | """ |
| 277 | if verbose: |
| 278 | logging.basicConfig(level=logging.DEBUG) |
| 279 | else: |
| 280 | logging.basicConfig(level=logging.INFO) |
| 281 | |
| 282 | |
| 283 | def _missing_environment_variables(): |
| 284 | """Verify the local environment has been set up to run atest. |
| 285 | |
| 286 | Returns: |
| 287 | List of strings of any missing environment variables. |
| 288 | """ |
| 289 | missing = filter(None, [x for x in EXPECTED_VARS if not os.environ.get(x)]) |
| 290 | if missing: |
| 291 | logging.error('Local environment doesn\'t appear to have been ' |
| 292 | 'initialized. Did you remember to run lunch? Expected ' |
| 293 | 'Environment Variables: %s.', missing) |
| 294 | return missing |
| 295 | |
| 296 | |
mikehoran | 95091b2 | 2017-10-31 15:55:26 -0700 | [diff] [blame] | 297 | def make_test_run_dir(): |
| 298 | """Make the test run dir in tmp. |
| 299 | |
| 300 | Returns: |
| 301 | A string of the dir path. |
| 302 | """ |
| 303 | utc_epoch_time = int(time.time()) |
| 304 | prefix = TEST_RUN_DIR_PREFIX % utc_epoch_time |
| 305 | return tempfile.mkdtemp(prefix=prefix) |
| 306 | |
| 307 | |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 308 | def run_tests(run_commands): |
| 309 | """Shell out and execute tradefed run commands. |
| 310 | |
| 311 | Args: |
| 312 | run_commands: A list of strings of Tradefed run commands. |
| 313 | """ |
| 314 | logging.info('Running tests') |
| 315 | # TODO: Build result parser for run command. Until then display raw stdout. |
| 316 | for run_command in run_commands: |
| 317 | logging.debug('Executing command: %s', run_command) |
| 318 | subprocess.check_call(run_command, shell=True, stderr=subprocess.STDOUT) |
| 319 | |
| 320 | |
Kevin Cheng | 7edb0b9 | 2017-12-14 15:00:25 -0800 | [diff] [blame] | 321 | def get_extra_args(args): |
| 322 | """Get extra args for test runners. |
| 323 | |
| 324 | Args: |
| 325 | args: arg parsed object. |
| 326 | |
| 327 | Returns: |
| 328 | Dict of extra args for test runners to utilize. |
| 329 | """ |
| 330 | extra_args = {} |
| 331 | if args.wait_for_debugger: |
| 332 | extra_args[constants.WAIT_FOR_DEBUGGER] = None |
| 333 | steps = args.steps or ALL_STEPS |
| 334 | if INSTALL_STEP not in steps: |
| 335 | extra_args[constants.DISABLE_INSTALL] = None |
Mike Ma | 150a61d | 2017-12-15 10:53:35 -0800 | [diff] [blame] | 336 | if args.generate_baseline: |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame^] | 337 | extra_args[constants.PRE_PATCH_ITERATIONS] = args.generate_baseline |
Mike Ma | 150a61d | 2017-12-15 10:53:35 -0800 | [diff] [blame] | 338 | if args.generate_new_metrics: |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame^] | 339 | extra_args[constants.POST_PATCH_ITERATIONS] = args.generate_new_metrics |
Kevin Cheng | 7edb0b9 | 2017-12-14 15:00:25 -0800 | [diff] [blame] | 340 | return extra_args |
| 341 | |
| 342 | |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame^] | 343 | def _get_regression_detection_args(args, results_dir): |
| 344 | """Get args for regression detection test runners. |
| 345 | |
| 346 | Args: |
| 347 | args: parsed args object. |
| 348 | results_dir: string directory to store atest results. |
| 349 | |
| 350 | Returns: |
| 351 | Dict of args for regression detection test runner to utilize. |
| 352 | """ |
| 353 | regression_args = {} |
| 354 | pre_patch_folder = (os.path.join(results_dir, 'baseline-metrics') if args.generate_baseline |
| 355 | else args.detect_regression.pop(0)) |
| 356 | post_patch_folder = (os.path.join(results_dir, 'new-metrics') if args.generate_new_metrics |
| 357 | else args.detect_regression.pop(0)) |
| 358 | regression_args[constants.PRE_PATCH_FOLDER] = pre_patch_folder |
| 359 | regression_args[constants.POST_PATCH_FOLDER] = post_patch_folder |
| 360 | return regression_args |
| 361 | |
| 362 | |
| 363 | def _will_run_tests(args): |
| 364 | """Determine if there are tests to run. |
| 365 | |
| 366 | Currently only used by detect_regression to skip the test if just running regression detection. |
| 367 | |
| 368 | Args: |
| 369 | args: parsed args object. |
| 370 | |
| 371 | Returns: |
| 372 | True if there are tests to run, false otherwise. |
| 373 | """ |
| 374 | return not (args.detect_regression and len(args.detect_regression) == 2) |
| 375 | |
| 376 | |
| 377 | def _has_valid_regression_detection_args(args): |
| 378 | """Validate regression detection args. |
| 379 | |
| 380 | Args: |
| 381 | args: parsed args object. |
| 382 | |
| 383 | Returns: |
| 384 | True if args are valid |
| 385 | """ |
| 386 | if args.generate_baseline and args.generate_new_metrics: |
| 387 | logging.error('Cannot collect both baseline and new metrics at the same time.') |
| 388 | return False |
| 389 | if args.detect_regression is not None: |
| 390 | if not args.detect_regression: |
| 391 | logging.error('Need to specify at least 1 arg for regression detection.') |
| 392 | return False |
| 393 | elif len(args.detect_regression) == 1: |
| 394 | if args.generate_baseline or args.generate_new_metrics: |
| 395 | return True |
| 396 | logging.error('Need to specify --generate-baseline or --generate-new-metrics.') |
| 397 | return False |
| 398 | elif len(args.detect_regression) == 2: |
| 399 | if args.generate_baseline: |
| 400 | logging.error('Specified 2 metric paths and --generate-baseline, ' |
| 401 | 'either drop --generate-baseline or drop a path') |
| 402 | return False |
| 403 | if args.generate_new_metrics: |
| 404 | logging.error('Specified 2 metric paths and --generate-new-metrics, ' |
| 405 | 'either drop --generate-new-metrics or drop a path') |
| 406 | return False |
| 407 | return True |
| 408 | else: |
| 409 | logging.error('Specified more than 2 metric paths.') |
| 410 | return False |
| 411 | return True |
| 412 | |
| 413 | |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 414 | def main(argv): |
mikehoran | 63d61b4 | 2017-07-28 15:28:50 -0700 | [diff] [blame] | 415 | """Entry point of atest script. |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 416 | |
mikehoran | 63d61b4 | 2017-07-28 15:28:50 -0700 | [diff] [blame] | 417 | Args: |
| 418 | argv: A list of arguments. |
Kevin Cheng | 09c2a2c | 2017-12-15 12:52:46 -0800 | [diff] [blame] | 419 | |
| 420 | Returns: |
| 421 | Exit code. |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 422 | """ |
mikehoran | 63d61b4 | 2017-07-28 15:28:50 -0700 | [diff] [blame] | 423 | args = _parse_args(argv) |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 424 | _configure_logging(args.verbose) |
| 425 | if _missing_environment_variables(): |
Dan Shi | fa016d1 | 2018-02-02 00:37:19 -0800 | [diff] [blame] | 426 | return constants.EXIT_CODE_ENV_NOT_SETUP |
Mike Ma | 150a61d | 2017-12-15 10:53:35 -0800 | [diff] [blame] | 427 | if args.generate_baseline and args.generate_new_metrics: |
| 428 | logging.error('Cannot collect both baseline and new metrics at the same time.') |
Dan Shi | fa016d1 | 2018-02-02 00:37:19 -0800 | [diff] [blame] | 429 | return constants.EXIT_CODE_ERROR |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame^] | 430 | if not _has_valid_regression_detection_args(args): |
| 431 | return constants.EXIT_CODE_ERROR |
Simran Basi | cf2189b | 2017-11-06 23:40:24 -0800 | [diff] [blame] | 432 | repo_root = os.environ.get(atest_utils.ANDROID_BUILD_TOP) |
mikehoran | 95091b2 | 2017-10-31 15:55:26 -0700 | [diff] [blame] | 433 | results_dir = make_test_run_dir() |
Simran Basi | 05384b8 | 2018-01-03 16:19:30 -0800 | [diff] [blame] | 434 | translator = cli_translator.CLITranslator( |
| 435 | results_dir=results_dir, root_dir=repo_root, |
| 436 | force_init=args.rebuild_module_info) |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame^] | 437 | build_targets = set() |
| 438 | test_infos = set() |
| 439 | if _will_run_tests(args): |
| 440 | try: |
| 441 | build_targets, test_infos = translator.translate(args.tests) |
| 442 | except cli_translator.TestDiscoveryException: |
| 443 | logging.exception('Error occured in test discovery:') |
| 444 | logging.info('This can happen after a repo sync or if the test is ' |
| 445 | 'new. Running: with "%s" may resolve the issue.', |
| 446 | REBUILD_MODULE_INFO_FLAG) |
| 447 | return constants.EXIT_CODE_TEST_NOT_FOUND |
Kevin Cheng | 7edb0b9 | 2017-12-14 15:00:25 -0800 | [diff] [blame] | 448 | build_targets |= test_runner_handler.get_test_runner_reqs(test_infos) |
| 449 | extra_args = get_extra_args(args) |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame^] | 450 | if args.detect_regression: |
| 451 | build_targets |= (regression_test_runner.RegressionTestRunner('') |
| 452 | .get_test_runner_build_reqs()) |
mikehoran | c327dca | 2017-11-27 16:24:22 -0800 | [diff] [blame] | 453 | # args.steps will be None if none of -bit set, else list of params set. |
| 454 | steps = args.steps if args.steps else ALL_STEPS |
| 455 | if BUILD_STEP in steps: |
mikehoran | c80dc53 | 2017-11-14 14:30:06 -0800 | [diff] [blame] | 456 | success = atest_utils.build(build_targets, args.verbose) |
| 457 | if not success: |
Dan Shi | fa016d1 | 2018-02-02 00:37:19 -0800 | [diff] [blame] | 458 | return constants.EXIT_CODE_BUILD_FAILURE |
mikehoran | c327dca | 2017-11-27 16:24:22 -0800 | [diff] [blame] | 459 | elif TEST_STEP not in steps: |
| 460 | logging.warn('Install step without test step currently not ' |
| 461 | 'supported, installing AND testing instead.') |
| 462 | steps.append(TEST_STEP) |
| 463 | if TEST_STEP in steps: |
Kevin Cheng | 7edb0b9 | 2017-12-14 15:00:25 -0800 | [diff] [blame] | 464 | test_runner_handler.run_all_tests(results_dir, test_infos, extra_args) |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame^] | 465 | if args.detect_regression: |
| 466 | regression_args = _get_regression_detection_args(args, results_dir) |
| 467 | regression_test_runner.RegressionTestRunner('').run_tests(None, regression_args) |
Dan Shi | fa016d1 | 2018-02-02 00:37:19 -0800 | [diff] [blame] | 468 | return constants.EXIT_CODE_SUCCESS |
mikehoran | 63d61b4 | 2017-07-28 15:28:50 -0700 | [diff] [blame] | 469 | |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 470 | if __name__ == '__main__': |
| 471 | sys.exit(main(sys.argv[1:])) |