Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # |
| 3 | # Copyright 2017, The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 17 | """ |
| 18 | Command line utility for running Android tests through TradeFederation. |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 19 | |
| 20 | atest helps automate the flow of building test modules across the Android |
| 21 | code base and executing the tests via the TradeFederation test harness. |
| 22 | |
| 23 | atest is designed to support any test types that can be ran by TradeFederation. |
| 24 | """ |
| 25 | |
nelsonli | edbd745 | 2018-08-27 11:11:11 +0800 | [diff] [blame] | 26 | from __future__ import print_function |
| 27 | |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 28 | import logging |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 29 | import os |
| 30 | import sys |
mikehoran | 95091b2 | 2017-10-31 15:55:26 -0700 | [diff] [blame] | 31 | import tempfile |
| 32 | import time |
kellyhung | 792fbcf | 2018-11-19 16:25:50 +0800 | [diff] [blame] | 33 | import platform |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 34 | |
Jim Tang | 6ed753e | 2019-07-23 10:39:58 +0800 | [diff] [blame] | 35 | from multiprocessing import Process |
| 36 | |
Jim Tang | 815b889 | 2018-07-11 12:57:30 +0800 | [diff] [blame] | 37 | import atest_arg_parser |
yangbill | bac1dd6 | 2019-06-03 17:06:40 +0800 | [diff] [blame] | 38 | import atest_error |
yelinhsieh | 4d5917d | 2019-03-12 17:26:27 +0800 | [diff] [blame] | 39 | import atest_execution_info |
Simran Basi | cf2189b | 2017-11-06 23:40:24 -0800 | [diff] [blame] | 40 | import atest_utils |
kellyhung | 7d004bb | 2019-04-02 11:54:59 +0800 | [diff] [blame] | 41 | import bug_detector |
mikehoran | 63d61b4 | 2017-07-28 15:28:50 -0700 | [diff] [blame] | 42 | import cli_translator |
Kevin Cheng | 7edb0b9 | 2017-12-14 15:00:25 -0800 | [diff] [blame] | 43 | # pylint: disable=import-error |
| 44 | import constants |
Kevin Cheng | 8b2c94c | 2017-12-18 14:43:26 -0800 | [diff] [blame] | 45 | import module_info |
mikehoran | 9b6b44b | 2018-04-09 15:54:58 -0700 | [diff] [blame] | 46 | import result_reporter |
Kevin Cheng | 7edb0b9 | 2017-12-14 15:00:25 -0800 | [diff] [blame] | 47 | import test_runner_handler |
kellyhung | 924b883 | 2019-03-05 18:35:00 +0800 | [diff] [blame] | 48 | |
| 49 | from metrics import metrics |
kellyhung | e3fa175 | 2019-04-23 11:13:41 +0800 | [diff] [blame] | 50 | from metrics import metrics_base |
kellyhung | 924b883 | 2019-03-05 18:35:00 +0800 | [diff] [blame] | 51 | from metrics import metrics_utils |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame] | 52 | from test_runners import regression_test_runner |
Jim Tang | 6ed753e | 2019-07-23 10:39:58 +0800 | [diff] [blame] | 53 | from tools import atest_tools |
Simran Basi | cf2189b | 2017-11-06 23:40:24 -0800 | [diff] [blame] | 54 | |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 55 | EXPECTED_VARS = frozenset([ |
Kevin Cheng | 8b2c94c | 2017-12-18 14:43:26 -0800 | [diff] [blame] | 56 | constants.ANDROID_BUILD_TOP, |
mikehoran | 43ed32d | 2017-08-18 17:13:36 -0700 | [diff] [blame] | 57 | 'ANDROID_TARGET_OUT_TESTCASES', |
Kevin Cheng | 8b2c94c | 2017-12-18 14:43:26 -0800 | [diff] [blame] | 58 | constants.ANDROID_OUT]) |
yelinhsieh | ebe0171 | 2019-08-15 11:24:37 +0800 | [diff] [blame] | 59 | TEST_RUN_DIR_PREFIX = "%Y%m%d_%H%M" |
Kevin Cheng | 21ea910 | 2018-02-22 10:52:42 -0800 | [diff] [blame] | 60 | CUSTOM_ARG_FLAG = '--' |
Dan Shi | 0ddd3e4 | 2018-05-30 11:24:30 -0700 | [diff] [blame] | 61 | OPTION_NOT_FOR_TEST_MAPPING = ( |
| 62 | 'Option `%s` does not work for running tests in TEST_MAPPING files') |
mikehoran | c80dc53 | 2017-11-14 14:30:06 -0800 | [diff] [blame] | 63 | |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 64 | DEVICE_TESTS = 'tests that require device' |
| 65 | HOST_TESTS = 'tests that do NOT require device' |
| 66 | RESULT_HEADER_FMT = '\nResults from %(test_type)s:' |
| 67 | RUN_HEADER_FMT = '\nRunning %(test_count)d %(test_type)s.' |
| 68 | TEST_COUNT = 'test_count' |
| 69 | TEST_TYPE = 'test_type' |
Jim Tang | 6ed753e | 2019-07-23 10:39:58 +0800 | [diff] [blame] | 70 | # Tasks that must run in the build time but unable to build by soong. |
| 71 | # (e.g subprocesses that invoke host commands.) |
| 72 | EXTRA_TASKS = { |
| 73 | 'index-targets': atest_tools.index_targets |
| 74 | } |
| 75 | |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 76 | |
Jim Tang | d1bc929 | 2019-09-05 11:54:20 +0800 | [diff] [blame] | 77 | def _run_extra_tasks(join=False): |
| 78 | """Execute EXTRA_TASKS with multiprocessing. |
| 79 | |
| 80 | Args: |
| 81 | join: A boolean that indicates the process should terminate when |
| 82 | the main process ends or keep itself alive. True indicates the |
| 83 | main process will wait for all subprocesses finish while False represents |
| 84 | killing all subprocesses when the main process exits. |
| 85 | """ |
| 86 | _running_procs = [] |
| 87 | for task in EXTRA_TASKS.values(): |
| 88 | proc = Process(target=task) |
| 89 | proc.daemon = not join |
| 90 | proc.start() |
| 91 | _running_procs.append(proc) |
| 92 | if join: |
| 93 | for proc in _running_procs: |
| 94 | proc.join() |
| 95 | |
| 96 | |
mikehoran | 63d61b4 | 2017-07-28 15:28:50 -0700 | [diff] [blame] | 97 | def _parse_args(argv): |
| 98 | """Parse command line arguments. |
| 99 | |
| 100 | Args: |
| 101 | argv: A list of arguments. |
| 102 | |
| 103 | Returns: |
| 104 | An argspace.Namespace class instance holding parsed args. |
| 105 | """ |
Kevin Cheng | 21ea910 | 2018-02-22 10:52:42 -0800 | [diff] [blame] | 106 | # Store everything after '--' in custom_args. |
| 107 | pruned_argv = argv |
| 108 | custom_args_index = None |
| 109 | if CUSTOM_ARG_FLAG in argv: |
| 110 | custom_args_index = argv.index(CUSTOM_ARG_FLAG) |
| 111 | pruned_argv = argv[:custom_args_index] |
Jim Tang | 815b889 | 2018-07-11 12:57:30 +0800 | [diff] [blame] | 112 | parser = atest_arg_parser.AtestArgParser() |
| 113 | parser.add_atest_args() |
Kevin Cheng | 21ea910 | 2018-02-22 10:52:42 -0800 | [diff] [blame] | 114 | args = parser.parse_args(pruned_argv) |
| 115 | args.custom_args = [] |
| 116 | if custom_args_index is not None: |
| 117 | args.custom_args = argv[custom_args_index+1:] |
| 118 | return args |
mikehoran | 63d61b4 | 2017-07-28 15:28:50 -0700 | [diff] [blame] | 119 | |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 120 | |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 121 | def _configure_logging(verbose): |
| 122 | """Configure the logger. |
| 123 | |
| 124 | Args: |
| 125 | verbose: A boolean. If true display DEBUG level logs. |
| 126 | """ |
mikehoran | b240182 | 2018-08-16 12:01:40 -0700 | [diff] [blame] | 127 | log_format = '%(asctime)s %(filename)s:%(lineno)s:%(levelname)s: %(message)s' |
| 128 | datefmt = '%Y-%m-%d %H:%M:%S' |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 129 | if verbose: |
mikehoran | b240182 | 2018-08-16 12:01:40 -0700 | [diff] [blame] | 130 | logging.basicConfig(level=logging.DEBUG, format=log_format, datefmt=datefmt) |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 131 | else: |
mikehoran | b240182 | 2018-08-16 12:01:40 -0700 | [diff] [blame] | 132 | logging.basicConfig(level=logging.INFO, format=log_format, datefmt=datefmt) |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 133 | |
| 134 | |
| 135 | def _missing_environment_variables(): |
| 136 | """Verify the local environment has been set up to run atest. |
| 137 | |
| 138 | Returns: |
| 139 | List of strings of any missing environment variables. |
| 140 | """ |
| 141 | missing = filter(None, [x for x in EXPECTED_VARS if not os.environ.get(x)]) |
| 142 | if missing: |
| 143 | logging.error('Local environment doesn\'t appear to have been ' |
| 144 | 'initialized. Did you remember to run lunch? Expected ' |
| 145 | 'Environment Variables: %s.', missing) |
| 146 | return missing |
| 147 | |
| 148 | |
mikehoran | 95091b2 | 2017-10-31 15:55:26 -0700 | [diff] [blame] | 149 | def make_test_run_dir(): |
yelinhsieh | ebe0171 | 2019-08-15 11:24:37 +0800 | [diff] [blame] | 150 | """Make the test run dir in ATEST_RESULT_ROOT. |
mikehoran | 95091b2 | 2017-10-31 15:55:26 -0700 | [diff] [blame] | 151 | |
| 152 | Returns: |
| 153 | A string of the dir path. |
| 154 | """ |
yelinhsieh | ebe0171 | 2019-08-15 11:24:37 +0800 | [diff] [blame] | 155 | if not os.path.exists(constants.ATEST_RESULT_ROOT): |
| 156 | os.makedirs(constants.ATEST_RESULT_ROOT) |
| 157 | ctime = time.strftime(TEST_RUN_DIR_PREFIX, time.localtime()) |
Jim Tang | 0fb6d11 | 2020-02-19 12:18:41 +0800 | [diff] [blame] | 158 | test_result_dir = tempfile.mkdtemp(prefix='%s_' % ctime, |
| 159 | dir=constants.ATEST_RESULT_ROOT) |
Jim Tang | 0fb6d11 | 2020-02-19 12:18:41 +0800 | [diff] [blame] | 160 | return test_result_dir |
mikehoran | 95091b2 | 2017-10-31 15:55:26 -0700 | [diff] [blame] | 161 | |
| 162 | |
Kevin Cheng | 7edb0b9 | 2017-12-14 15:00:25 -0800 | [diff] [blame] | 163 | def get_extra_args(args): |
| 164 | """Get extra args for test runners. |
| 165 | |
| 166 | Args: |
| 167 | args: arg parsed object. |
| 168 | |
| 169 | Returns: |
| 170 | Dict of extra args for test runners to utilize. |
| 171 | """ |
| 172 | extra_args = {} |
| 173 | if args.wait_for_debugger: |
| 174 | extra_args[constants.WAIT_FOR_DEBUGGER] = None |
Jim Tang | 815b889 | 2018-07-11 12:57:30 +0800 | [diff] [blame] | 175 | steps = args.steps or constants.ALL_STEPS |
| 176 | if constants.INSTALL_STEP not in steps: |
Kevin Cheng | 7edb0b9 | 2017-12-14 15:00:25 -0800 | [diff] [blame] | 177 | extra_args[constants.DISABLE_INSTALL] = None |
yelinhsieh | 7be4a7f | 2019-08-15 14:27:22 +0800 | [diff] [blame] | 178 | # The key and its value of the dict can be called via: |
| 179 | # if args.aaaa: |
| 180 | # extra_args[constants.AAAA] = args.aaaa |
| 181 | arg_maps = {'all_abi': constants.ALL_ABI, |
kellyhung | a118aa5 | 2020-01-13 12:09:03 +0800 | [diff] [blame] | 182 | 'collect_tests_only': constants.COLLECT_TESTS_ONLY, |
yelinhsieh | 7be4a7f | 2019-08-15 14:27:22 +0800 | [diff] [blame] | 183 | 'custom_args': constants.CUSTOM_ARGS, |
| 184 | 'disable_teardown': constants.DISABLE_TEARDOWN, |
| 185 | 'dry_run': constants.DRY_RUN, |
| 186 | 'generate_baseline': constants.PRE_PATCH_ITERATIONS, |
| 187 | 'generate_new_metrics': constants.POST_PATCH_ITERATIONS, |
| 188 | 'host': constants.HOST, |
| 189 | 'instant': constants.INSTANT, |
kellyhung | 3e5f8a9 | 2019-08-08 16:02:55 +0800 | [diff] [blame] | 190 | 'iterations': constants.ITERATIONS, |
| 191 | 'rerun_until_failure': constants.RERUN_UNTIL_FAILURE, |
| 192 | 'retry_any_failure': constants.RETRY_ANY_FAILURE, |
yelinhsieh | 7be4a7f | 2019-08-15 14:27:22 +0800 | [diff] [blame] | 193 | 'serial': constants.SERIAL, |
yelinhsieh | 8413afc | 2020-02-20 18:02:41 +0800 | [diff] [blame] | 194 | 'sharding': constants.SHARDING, |
yelinhsieh | 804ad2a | 2020-02-20 16:31:05 +0800 | [diff] [blame] | 195 | 'tf_debug': constants.TF_DEBUG, |
yangbill | b309a89 | 2019-12-26 18:19:52 +0800 | [diff] [blame] | 196 | 'tf_template': constants.TF_TEMPLATE, |
yelinhsieh | 7be4a7f | 2019-08-15 14:27:22 +0800 | [diff] [blame] | 197 | 'user_type': constants.USER_TYPE} |
| 198 | not_match = [k for k in arg_maps if k not in vars(args)] |
| 199 | if not_match: |
| 200 | raise AttributeError('%s object has no attribute %s' |
| 201 | %(type(args).__name__, not_match)) |
| 202 | extra_args.update({arg_maps.get(k): v for k, v in vars(args).items() |
| 203 | if arg_maps.get(k) and v}) |
Kevin Cheng | 7edb0b9 | 2017-12-14 15:00:25 -0800 | [diff] [blame] | 204 | return extra_args |
| 205 | |
| 206 | |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame] | 207 | def _get_regression_detection_args(args, results_dir): |
| 208 | """Get args for regression detection test runners. |
| 209 | |
| 210 | Args: |
| 211 | args: parsed args object. |
| 212 | results_dir: string directory to store atest results. |
| 213 | |
| 214 | Returns: |
| 215 | Dict of args for regression detection test runner to utilize. |
| 216 | """ |
| 217 | regression_args = {} |
| 218 | pre_patch_folder = (os.path.join(results_dir, 'baseline-metrics') if args.generate_baseline |
| 219 | else args.detect_regression.pop(0)) |
| 220 | post_patch_folder = (os.path.join(results_dir, 'new-metrics') if args.generate_new_metrics |
| 221 | else args.detect_regression.pop(0)) |
| 222 | regression_args[constants.PRE_PATCH_FOLDER] = pre_patch_folder |
| 223 | regression_args[constants.POST_PATCH_FOLDER] = post_patch_folder |
| 224 | return regression_args |
| 225 | |
| 226 | |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 227 | def _validate_exec_mode(args, test_infos, host_tests=None): |
kellyhung | 0625d17 | 2018-06-21 16:40:27 +0800 | [diff] [blame] | 228 | """Validate all test execution modes are not in conflict. |
| 229 | |
| 230 | Exit the program with error code if have device-only and host-only. |
| 231 | If no conflict and host side, add args.host=True. |
| 232 | |
| 233 | Args: |
| 234 | args: parsed args object. |
| 235 | test_info: TestInfo object. |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 236 | host_tests: True if all tests should be deviceless, False if all tests |
| 237 | should be device tests. Default is set to None, which means |
| 238 | tests can be either deviceless or device tests. |
kellyhung | 0625d17 | 2018-06-21 16:40:27 +0800 | [diff] [blame] | 239 | """ |
| 240 | all_device_modes = [x.get_supported_exec_mode() for x in test_infos] |
kellyhung | 924b883 | 2019-03-05 18:35:00 +0800 | [diff] [blame] | 241 | err_msg = None |
kellyhung | 0625d17 | 2018-06-21 16:40:27 +0800 | [diff] [blame] | 242 | # In the case of '$atest <device-only> --host', exit. |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 243 | if (host_tests or args.host) and constants.DEVICE_TEST in all_device_modes: |
kellyhung | 0625d17 | 2018-06-21 16:40:27 +0800 | [diff] [blame] | 244 | err_msg = ('Test side and option(--host) conflict. Please remove ' |
| 245 | '--host if the test run on device side.') |
kellyhung | 0625d17 | 2018-06-21 16:40:27 +0800 | [diff] [blame] | 246 | # In the case of '$atest <host-only> <device-only> --host' or |
| 247 | # '$atest <host-only> <device-only>', exit. |
| 248 | if (constants.DEVICELESS_TEST in all_device_modes and |
| 249 | constants.DEVICE_TEST in all_device_modes): |
| 250 | err_msg = 'There are host-only and device-only tests in command.' |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 251 | if host_tests is False and constants.DEVICELESS_TEST in all_device_modes: |
| 252 | err_msg = 'There are host-only tests in command.' |
kellyhung | 924b883 | 2019-03-05 18:35:00 +0800 | [diff] [blame] | 253 | if err_msg: |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 254 | logging.error(err_msg) |
kellyhung | 924b883 | 2019-03-05 18:35:00 +0800 | [diff] [blame] | 255 | metrics_utils.send_exit_event(constants.EXIT_CODE_ERROR, logs=err_msg) |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 256 | sys.exit(constants.EXIT_CODE_ERROR) |
kellyhung | 0625d17 | 2018-06-21 16:40:27 +0800 | [diff] [blame] | 257 | # In the case of '$atest <host-only>', we add --host to run on host-side. |
Jim Tang | b047780 | 2019-11-26 10:20:09 +0800 | [diff] [blame] | 258 | # The option should only be overridden if `host_tests` is not set. |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 259 | if not args.host and host_tests is None: |
kellyhung | 0625d17 | 2018-06-21 16:40:27 +0800 | [diff] [blame] | 260 | args.host = bool(constants.DEVICELESS_TEST in all_device_modes) |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 261 | |
| 262 | |
| 263 | def _validate_tm_tests_exec_mode(args, test_infos): |
| 264 | """Validate all test execution modes are not in conflict. |
| 265 | |
| 266 | Split the tests in Test Mapping files into two groups, device tests and |
| 267 | deviceless tests running on host. Validate the tests' host setting. |
| 268 | For device tests, exit the program if any test is found for host-only. |
| 269 | For deviceless tests, exit the program if any test is found for device-only. |
| 270 | |
| 271 | Args: |
| 272 | args: parsed args object. |
| 273 | test_info: TestInfo object. |
| 274 | """ |
| 275 | device_test_infos, host_test_infos = _split_test_mapping_tests( |
| 276 | test_infos) |
| 277 | # No need to verify device tests if atest command is set to only run host |
| 278 | # tests. |
| 279 | if device_test_infos and not args.host: |
| 280 | _validate_exec_mode(args, device_test_infos, host_tests=False) |
| 281 | if host_test_infos: |
| 282 | _validate_exec_mode(args, host_test_infos, host_tests=True) |
| 283 | |
kellyhung | 0625d17 | 2018-06-21 16:40:27 +0800 | [diff] [blame] | 284 | |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame] | 285 | def _will_run_tests(args): |
| 286 | """Determine if there are tests to run. |
| 287 | |
| 288 | Currently only used by detect_regression to skip the test if just running regression detection. |
| 289 | |
| 290 | Args: |
| 291 | args: parsed args object. |
| 292 | |
| 293 | Returns: |
| 294 | True if there are tests to run, false otherwise. |
| 295 | """ |
| 296 | return not (args.detect_regression and len(args.detect_regression) == 2) |
| 297 | |
| 298 | |
| 299 | def _has_valid_regression_detection_args(args): |
| 300 | """Validate regression detection args. |
| 301 | |
| 302 | Args: |
| 303 | args: parsed args object. |
| 304 | |
| 305 | Returns: |
| 306 | True if args are valid |
| 307 | """ |
| 308 | if args.generate_baseline and args.generate_new_metrics: |
| 309 | logging.error('Cannot collect both baseline and new metrics at the same time.') |
| 310 | return False |
| 311 | if args.detect_regression is not None: |
| 312 | if not args.detect_regression: |
| 313 | logging.error('Need to specify at least 1 arg for regression detection.') |
| 314 | return False |
| 315 | elif len(args.detect_regression) == 1: |
| 316 | if args.generate_baseline or args.generate_new_metrics: |
| 317 | return True |
| 318 | logging.error('Need to specify --generate-baseline or --generate-new-metrics.') |
| 319 | return False |
| 320 | elif len(args.detect_regression) == 2: |
| 321 | if args.generate_baseline: |
| 322 | logging.error('Specified 2 metric paths and --generate-baseline, ' |
| 323 | 'either drop --generate-baseline or drop a path') |
| 324 | return False |
| 325 | if args.generate_new_metrics: |
| 326 | logging.error('Specified 2 metric paths and --generate-new-metrics, ' |
| 327 | 'either drop --generate-new-metrics or drop a path') |
| 328 | return False |
| 329 | return True |
| 330 | else: |
| 331 | logging.error('Specified more than 2 metric paths.') |
| 332 | return False |
| 333 | return True |
| 334 | |
| 335 | |
Dan Shi | 0ddd3e4 | 2018-05-30 11:24:30 -0700 | [diff] [blame] | 336 | def _has_valid_test_mapping_args(args): |
| 337 | """Validate test mapping args. |
| 338 | |
| 339 | Not all args work when running tests in TEST_MAPPING files. Validate the |
| 340 | args before running the tests. |
| 341 | |
| 342 | Args: |
| 343 | args: parsed args object. |
| 344 | |
| 345 | Returns: |
| 346 | True if args are valid |
| 347 | """ |
| 348 | is_test_mapping = atest_utils.is_test_mapping(args) |
| 349 | if not is_test_mapping: |
| 350 | return True |
| 351 | options_to_validate = [ |
| 352 | (args.generate_baseline, '--generate-baseline'), |
| 353 | (args.detect_regression, '--detect-regression'), |
| 354 | (args.generate_new_metrics, '--generate-new-metrics'), |
| 355 | ] |
| 356 | for arg_value, arg in options_to_validate: |
| 357 | if arg_value: |
| 358 | logging.error(OPTION_NOT_FOR_TEST_MAPPING, arg) |
| 359 | return False |
| 360 | return True |
| 361 | |
| 362 | |
Dan Shi | e4e267f | 2018-06-01 11:31:57 -0700 | [diff] [blame] | 363 | def _validate_args(args): |
| 364 | """Validate setups and args. |
| 365 | |
| 366 | Exit the program with error code if any setup or arg is invalid. |
| 367 | |
| 368 | Args: |
| 369 | args: parsed args object. |
| 370 | """ |
| 371 | if _missing_environment_variables(): |
| 372 | sys.exit(constants.EXIT_CODE_ENV_NOT_SETUP) |
| 373 | if args.generate_baseline and args.generate_new_metrics: |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 374 | logging.error( |
| 375 | 'Cannot collect both baseline and new metrics at the same time.') |
Dan Shi | e4e267f | 2018-06-01 11:31:57 -0700 | [diff] [blame] | 376 | sys.exit(constants.EXIT_CODE_ERROR) |
| 377 | if not _has_valid_regression_detection_args(args): |
| 378 | sys.exit(constants.EXIT_CODE_ERROR) |
| 379 | if not _has_valid_test_mapping_args(args): |
| 380 | sys.exit(constants.EXIT_CODE_ERROR) |
| 381 | |
nelsonli | e3f90de | 2018-06-22 14:59:39 +0800 | [diff] [blame] | 382 | |
| 383 | def _print_module_info_from_module_name(mod_info, module_name): |
| 384 | """print out the related module_info for a module_name. |
| 385 | |
| 386 | Args: |
| 387 | mod_info: ModuleInfo object. |
| 388 | module_name: A string of module. |
| 389 | |
| 390 | Returns: |
| 391 | True if the module_info is found. |
| 392 | """ |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 393 | title_mapping = { |
| 394 | constants.MODULE_PATH: "Source code path", |
| 395 | constants.MODULE_INSTALLED: "Installed path", |
| 396 | constants.MODULE_COMPATIBILITY_SUITES: "Compatibility suite"} |
nelsonli | e3f90de | 2018-06-22 14:59:39 +0800 | [diff] [blame] | 397 | target_module_info = mod_info.get_module_info(module_name) |
| 398 | is_module_found = False |
| 399 | if target_module_info: |
nelsonli | edbd745 | 2018-08-27 11:11:11 +0800 | [diff] [blame] | 400 | atest_utils.colorful_print(module_name, constants.GREEN) |
nelsonli | e3f90de | 2018-06-22 14:59:39 +0800 | [diff] [blame] | 401 | for title_key in title_mapping.iterkeys(): |
| 402 | atest_utils.colorful_print("\t%s" % title_mapping[title_key], |
nelsonli | edbd745 | 2018-08-27 11:11:11 +0800 | [diff] [blame] | 403 | constants.CYAN) |
nelsonli | e3f90de | 2018-06-22 14:59:39 +0800 | [diff] [blame] | 404 | for info_value in target_module_info[title_key]: |
nelsonli | edbd745 | 2018-08-27 11:11:11 +0800 | [diff] [blame] | 405 | print("\t\t{}".format(info_value)) |
nelsonli | e3f90de | 2018-06-22 14:59:39 +0800 | [diff] [blame] | 406 | is_module_found = True |
| 407 | return is_module_found |
| 408 | |
| 409 | |
| 410 | def _print_test_info(mod_info, test_infos): |
| 411 | """Print the module information from TestInfos. |
| 412 | |
| 413 | Args: |
| 414 | mod_info: ModuleInfo object. |
| 415 | test_infos: A list of TestInfos. |
| 416 | |
| 417 | Returns: |
| 418 | Always return EXIT_CODE_SUCCESS |
| 419 | """ |
| 420 | for test_info in test_infos: |
| 421 | _print_module_info_from_module_name(mod_info, test_info.test_name) |
nelsonli | edbd745 | 2018-08-27 11:11:11 +0800 | [diff] [blame] | 422 | atest_utils.colorful_print("\tRelated build targets", constants.MAGENTA) |
| 423 | print("\t\t{}".format(", ".join(test_info.build_targets))) |
nelsonli | e3f90de | 2018-06-22 14:59:39 +0800 | [diff] [blame] | 424 | for build_target in test_info.build_targets: |
| 425 | if build_target != test_info.test_name: |
| 426 | _print_module_info_from_module_name(mod_info, build_target) |
| 427 | atest_utils.colorful_print("", constants.WHITE) |
| 428 | return constants.EXIT_CODE_SUCCESS |
| 429 | |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 430 | |
| 431 | def is_from_test_mapping(test_infos): |
| 432 | """Check that the test_infos came from TEST_MAPPING files. |
| 433 | |
| 434 | Args: |
| 435 | test_infos: A set of TestInfos. |
| 436 | |
Jim Tang | b047780 | 2019-11-26 10:20:09 +0800 | [diff] [blame] | 437 | Returns: |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 438 | True if the test infos are from TEST_MAPPING files. |
| 439 | """ |
| 440 | return list(test_infos)[0].from_test_mapping |
| 441 | |
| 442 | |
| 443 | def _split_test_mapping_tests(test_infos): |
| 444 | """Split Test Mapping tests into 2 groups: device tests and host tests. |
| 445 | |
| 446 | Args: |
| 447 | test_infos: A set of TestInfos. |
| 448 | |
Jim Tang | b047780 | 2019-11-26 10:20:09 +0800 | [diff] [blame] | 449 | Returns: |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 450 | A tuple of (device_test_infos, host_test_infos), where |
| 451 | device_test_infos: A set of TestInfos for tests that require device. |
| 452 | host_test_infos: A set of TestInfos for tests that do NOT require |
| 453 | device. |
| 454 | """ |
| 455 | assert is_from_test_mapping(test_infos) |
| 456 | host_test_infos = set([info for info in test_infos if info.host]) |
| 457 | device_test_infos = set([info for info in test_infos if not info.host]) |
| 458 | return device_test_infos, host_test_infos |
| 459 | |
| 460 | |
| 461 | # pylint: disable=too-many-locals |
| 462 | def _run_test_mapping_tests(results_dir, test_infos, extra_args): |
| 463 | """Run all tests in TEST_MAPPING files. |
| 464 | |
| 465 | Args: |
| 466 | results_dir: String directory to store atest results. |
| 467 | test_infos: A set of TestInfos. |
| 468 | extra_args: Dict of extra args to add to test run. |
| 469 | |
| 470 | Returns: |
| 471 | Exit code. |
| 472 | """ |
| 473 | device_test_infos, host_test_infos = _split_test_mapping_tests(test_infos) |
| 474 | # `host` option needs to be set to True to run host side tests. |
| 475 | host_extra_args = extra_args.copy() |
| 476 | host_extra_args[constants.HOST] = True |
| 477 | test_runs = [(host_test_infos, host_extra_args, HOST_TESTS)] |
| 478 | if extra_args.get(constants.HOST): |
| 479 | atest_utils.colorful_print( |
| 480 | 'Option `--host` specified. Skip running device tests.', |
| 481 | constants.MAGENTA) |
| 482 | else: |
| 483 | test_runs.append((device_test_infos, extra_args, DEVICE_TESTS)) |
| 484 | |
| 485 | test_results = [] |
| 486 | for tests, args, test_type in test_runs: |
| 487 | if not tests: |
| 488 | continue |
| 489 | header = RUN_HEADER_FMT % {TEST_COUNT: len(tests), TEST_TYPE: test_type} |
| 490 | atest_utils.colorful_print(header, constants.MAGENTA) |
| 491 | logging.debug('\n'.join([str(info) for info in tests])) |
| 492 | tests_exit_code, reporter = test_runner_handler.run_all_tests( |
| 493 | results_dir, tests, args, delay_print_summary=True) |
yelinhsieh | 4d5917d | 2019-03-12 17:26:27 +0800 | [diff] [blame] | 494 | atest_execution_info.AtestExecutionInfo.result_reporters.append(reporter) |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 495 | test_results.append((tests_exit_code, reporter, test_type)) |
| 496 | |
| 497 | all_tests_exit_code = constants.EXIT_CODE_SUCCESS |
| 498 | failed_tests = [] |
| 499 | for tests_exit_code, reporter, test_type in test_results: |
| 500 | atest_utils.colorful_print( |
| 501 | RESULT_HEADER_FMT % {TEST_TYPE: test_type}, constants.MAGENTA) |
| 502 | result = tests_exit_code | reporter.print_summary() |
| 503 | if result: |
| 504 | failed_tests.append(test_type) |
| 505 | all_tests_exit_code |= result |
| 506 | |
| 507 | # List failed tests at the end as a reminder. |
| 508 | if failed_tests: |
| 509 | atest_utils.colorful_print( |
| 510 | '\n==============================', constants.YELLOW) |
| 511 | atest_utils.colorful_print( |
| 512 | '\nFollowing tests failed:', constants.MAGENTA) |
| 513 | for failure in failed_tests: |
| 514 | atest_utils.colorful_print(failure, constants.RED) |
| 515 | |
| 516 | return all_tests_exit_code |
| 517 | |
| 518 | |
yangbill | cc1a21f | 2018-12-12 20:03:12 +0800 | [diff] [blame] | 519 | def _dry_run(results_dir, extra_args, test_infos): |
| 520 | """Only print the commands of the target tests rather than running them in actual. |
| 521 | |
| 522 | Args: |
| 523 | results_dir: Path for saving atest logs. |
| 524 | extra_args: Dict of extra args for test runners to utilize. |
| 525 | test_infos: A list of TestInfos. |
yangbill | 52c63fa | 2019-05-24 09:55:00 +0800 | [diff] [blame] | 526 | |
| 527 | Returns: |
| 528 | A list of test commands. |
yangbill | cc1a21f | 2018-12-12 20:03:12 +0800 | [diff] [blame] | 529 | """ |
yangbill | 52c63fa | 2019-05-24 09:55:00 +0800 | [diff] [blame] | 530 | all_run_cmds = [] |
yangbill | cc1a21f | 2018-12-12 20:03:12 +0800 | [diff] [blame] | 531 | for test_runner, tests in test_runner_handler.group_tests_by_test_runners(test_infos): |
| 532 | runner = test_runner(results_dir) |
| 533 | run_cmds = runner.generate_run_commands(tests, extra_args) |
| 534 | for run_cmd in run_cmds: |
yangbill | 52c63fa | 2019-05-24 09:55:00 +0800 | [diff] [blame] | 535 | all_run_cmds.append(run_cmd) |
yangbill | cc1a21f | 2018-12-12 20:03:12 +0800 | [diff] [blame] | 536 | print('Would run test via command: %s' |
| 537 | % (atest_utils.colorize(run_cmd, constants.GREEN))) |
yangbill | 52c63fa | 2019-05-24 09:55:00 +0800 | [diff] [blame] | 538 | return all_run_cmds |
yangbill | cc1a21f | 2018-12-12 20:03:12 +0800 | [diff] [blame] | 539 | |
easoncylee | f0fb2b1 | 2019-01-22 15:49:09 +0800 | [diff] [blame] | 540 | def _print_testable_modules(mod_info, suite): |
| 541 | """Print the testable modules for a given suite. |
| 542 | |
| 543 | Args: |
| 544 | mod_info: ModuleInfo object. |
| 545 | suite: A string of suite name. |
| 546 | """ |
| 547 | testable_modules = mod_info.get_testable_modules(suite) |
| 548 | print('\n%s' % atest_utils.colorize('%s Testable %s modules' % ( |
| 549 | len(testable_modules), suite), constants.CYAN)) |
| 550 | print('-------') |
| 551 | for module in sorted(testable_modules): |
| 552 | print('\t%s' % module) |
yangbill | cc1a21f | 2018-12-12 20:03:12 +0800 | [diff] [blame] | 553 | |
Jim Tang | 4e056c6 | 2019-11-08 14:53:18 +0800 | [diff] [blame] | 554 | def _is_inside_android_root(): |
| 555 | """Identify whether the cwd is inside of Android source tree. |
| 556 | |
| 557 | Returns: |
| 558 | False if the cwd is outside of the source tree, True otherwise. |
| 559 | """ |
| 560 | build_top = os.getenv(constants.ANDROID_BUILD_TOP, ' ') |
| 561 | return build_top in os.getcwd() |
| 562 | |
kellyhung | e6a643c | 2018-12-19 11:31:25 +0800 | [diff] [blame] | 563 | # pylint: disable=too-many-statements |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 564 | # pylint: disable=too-many-branches |
yelinhsieh | 4d5917d | 2019-03-12 17:26:27 +0800 | [diff] [blame] | 565 | def main(argv, results_dir): |
mikehoran | 63d61b4 | 2017-07-28 15:28:50 -0700 | [diff] [blame] | 566 | """Entry point of atest script. |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 567 | |
mikehoran | 63d61b4 | 2017-07-28 15:28:50 -0700 | [diff] [blame] | 568 | Args: |
| 569 | argv: A list of arguments. |
yelinhsieh | 4d5917d | 2019-03-12 17:26:27 +0800 | [diff] [blame] | 570 | results_dir: A directory which stores the ATest execution information. |
Kevin Cheng | 09c2a2c | 2017-12-15 12:52:46 -0800 | [diff] [blame] | 571 | |
| 572 | Returns: |
| 573 | Exit code. |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 574 | """ |
mikehoran | 63d61b4 | 2017-07-28 15:28:50 -0700 | [diff] [blame] | 575 | args = _parse_args(argv) |
mikehoran | be9102f | 2017-08-04 16:04:03 -0700 | [diff] [blame] | 576 | _configure_logging(args.verbose) |
Dan Shi | e4e267f | 2018-06-01 11:31:57 -0700 | [diff] [blame] | 577 | _validate_args(args) |
kellyhung | 924b883 | 2019-03-05 18:35:00 +0800 | [diff] [blame] | 578 | metrics_utils.get_start_time() |
| 579 | metrics.AtestStartEvent( |
| 580 | command_line=' '.join(argv), |
| 581 | test_references=args.tests, |
| 582 | cwd=os.getcwd(), |
| 583 | os=platform.platform()) |
Jim Tang | 139389a | 2019-09-27 14:48:30 +0800 | [diff] [blame] | 584 | if args.version: |
| 585 | if os.path.isfile(constants.VERSION_FILE): |
| 586 | with open(constants.VERSION_FILE) as version_file: |
| 587 | print(version_file.read()) |
| 588 | return constants.EXIT_CODE_SUCCESS |
Jim Tang | 4e056c6 | 2019-11-08 14:53:18 +0800 | [diff] [blame] | 589 | if not _is_inside_android_root(): |
| 590 | atest_utils.colorful_print( |
| 591 | "\nAtest must always work under ${}!".format( |
| 592 | constants.ANDROID_BUILD_TOP), constants.RED) |
| 593 | return constants.EXIT_CODE_OUTSIDE_ROOT |
Jim Tang | 14397f0 | 2019-12-03 16:28:02 +0800 | [diff] [blame] | 594 | if args.help: |
| 595 | atest_arg_parser.print_epilog_text() |
| 596 | return constants.EXIT_CODE_SUCCESS |
Kevin Cheng | 8b2c94c | 2017-12-18 14:43:26 -0800 | [diff] [blame] | 597 | mod_info = module_info.ModuleInfo(force_build=args.rebuild_module_info) |
Jim Tang | d1bc929 | 2019-09-05 11:54:20 +0800 | [diff] [blame] | 598 | if args.rebuild_module_info: |
| 599 | _run_extra_tasks(join=True) |
Kevin Cheng | 8b2c94c | 2017-12-18 14:43:26 -0800 | [diff] [blame] | 600 | translator = cli_translator.CLITranslator(module_info=mod_info) |
easoncylee | f0fb2b1 | 2019-01-22 15:49:09 +0800 | [diff] [blame] | 601 | if args.list_modules: |
| 602 | _print_testable_modules(mod_info, args.list_modules) |
| 603 | return constants.EXIT_CODE_SUCCESS |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame] | 604 | build_targets = set() |
| 605 | test_infos = set() |
yangbill | 0b35e4b | 2019-06-10 20:36:28 +0800 | [diff] [blame] | 606 | # Clear cache if user pass -c option |
| 607 | if args.clear_cache: |
| 608 | atest_utils.clean_test_info_caches(args.tests) |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame] | 609 | if _will_run_tests(args): |
nelsonli | c4a7145 | 2018-09-13 14:10:30 +0800 | [diff] [blame] | 610 | build_targets, test_infos = translator.translate(args) |
| 611 | if not test_infos: |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame] | 612 | return constants.EXIT_CODE_TEST_NOT_FOUND |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 613 | if not is_from_test_mapping(test_infos): |
| 614 | _validate_exec_mode(args, test_infos) |
| 615 | else: |
| 616 | _validate_tm_tests_exec_mode(args, test_infos) |
nelsonli | e3f90de | 2018-06-22 14:59:39 +0800 | [diff] [blame] | 617 | if args.info: |
| 618 | return _print_test_info(mod_info, test_infos) |
Kevin Cheng | 8b2c94c | 2017-12-18 14:43:26 -0800 | [diff] [blame] | 619 | build_targets |= test_runner_handler.get_test_runner_reqs(mod_info, |
| 620 | test_infos) |
Kevin Cheng | 7edb0b9 | 2017-12-14 15:00:25 -0800 | [diff] [blame] | 621 | extra_args = get_extra_args(args) |
yangbill | bac1dd6 | 2019-06-03 17:06:40 +0800 | [diff] [blame] | 622 | if args.update_cmd_mapping or args.verify_cmd_mapping: |
yangbill | 52c63fa | 2019-05-24 09:55:00 +0800 | [diff] [blame] | 623 | args.dry_run = True |
yangbill | cc1a21f | 2018-12-12 20:03:12 +0800 | [diff] [blame] | 624 | if args.dry_run: |
yangbill | bac1dd6 | 2019-06-03 17:06:40 +0800 | [diff] [blame] | 625 | args.tests.sort() |
yangbill | 52c63fa | 2019-05-24 09:55:00 +0800 | [diff] [blame] | 626 | dry_run_cmds = _dry_run(results_dir, extra_args, test_infos) |
yangbill | bac1dd6 | 2019-06-03 17:06:40 +0800 | [diff] [blame] | 627 | if args.verify_cmd_mapping: |
| 628 | try: |
| 629 | atest_utils.handle_test_runner_cmd(' '.join(args.tests), |
| 630 | dry_run_cmds, |
| 631 | do_verification=True) |
| 632 | except atest_error.DryRunVerificationError as e: |
| 633 | atest_utils.colorful_print(str(e), constants.RED) |
| 634 | return constants.EXIT_CODE_VERIFY_FAILURE |
yangbill | 52c63fa | 2019-05-24 09:55:00 +0800 | [diff] [blame] | 635 | if args.update_cmd_mapping: |
yangbill | bac1dd6 | 2019-06-03 17:06:40 +0800 | [diff] [blame] | 636 | atest_utils.handle_test_runner_cmd(' '.join(args.tests), |
yangbill | 52c63fa | 2019-05-24 09:55:00 +0800 | [diff] [blame] | 637 | dry_run_cmds) |
yangbill | cc1a21f | 2018-12-12 20:03:12 +0800 | [diff] [blame] | 638 | return constants.EXIT_CODE_SUCCESS |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame] | 639 | if args.detect_regression: |
| 640 | build_targets |= (regression_test_runner.RegressionTestRunner('') |
| 641 | .get_test_runner_build_reqs()) |
mikehoran | c327dca | 2017-11-27 16:24:22 -0800 | [diff] [blame] | 642 | # args.steps will be None if none of -bit set, else list of params set. |
Jim Tang | 815b889 | 2018-07-11 12:57:30 +0800 | [diff] [blame] | 643 | steps = args.steps if args.steps else constants.ALL_STEPS |
| 644 | if build_targets and constants.BUILD_STEP in steps: |
Jim Tang | d1bc929 | 2019-09-05 11:54:20 +0800 | [diff] [blame] | 645 | if constants.TEST_STEP in steps and not args.rebuild_module_info: |
| 646 | # Run extra tasks along with build step concurrently. Note that |
| 647 | # Atest won't index targets when only "-b" is given(without -t). |
| 648 | _run_extra_tasks(join=False) |
Kevin Cheng | 5be930e | 2018-02-20 09:39:22 -0800 | [diff] [blame] | 649 | # Add module-info.json target to the list of build targets to keep the |
| 650 | # file up to date. |
| 651 | build_targets.add(mod_info.module_info_target) |
patricktu | 5456df0 | 2019-11-01 20:15:02 +0800 | [diff] [blame] | 652 | # Build the deps-license to generate dependencies data in |
| 653 | # module-info.json. |
| 654 | build_targets.add(constants.DEPS_LICENSE) |
patricktu | 5456df0 | 2019-11-01 20:15:02 +0800 | [diff] [blame] | 655 | # The environment variables PROJ_PATH and DEP_PATH are necessary for the |
| 656 | # deps-license. |
yangbill | 741dc12 | 2020-02-12 10:46:19 +0800 | [diff] [blame] | 657 | build_env = dict(constants.DEPS_LICENSE_ENV) |
kellyhung | 23c55b8 | 2019-01-04 16:58:14 +0800 | [diff] [blame] | 658 | build_start = time.time() |
Jim Tang | 6ed753e | 2019-07-23 10:39:58 +0800 | [diff] [blame] | 659 | success = atest_utils.build(build_targets, verbose=args.verbose, |
patricktu | 5456df0 | 2019-11-01 20:15:02 +0800 | [diff] [blame] | 660 | env_vars=build_env) |
kellyhung | 23c55b8 | 2019-01-04 16:58:14 +0800 | [diff] [blame] | 661 | metrics.BuildFinishEvent( |
| 662 | duration=metrics_utils.convert_duration(time.time() - build_start), |
| 663 | success=success, |
| 664 | targets=build_targets) |
mikehoran | c80dc53 | 2017-11-14 14:30:06 -0800 | [diff] [blame] | 665 | if not success: |
Dan Shi | fa016d1 | 2018-02-02 00:37:19 -0800 | [diff] [blame] | 666 | return constants.EXIT_CODE_BUILD_FAILURE |
Jim Tang | 815b889 | 2018-07-11 12:57:30 +0800 | [diff] [blame] | 667 | elif constants.TEST_STEP not in steps: |
mikehoran | c327dca | 2017-11-27 16:24:22 -0800 | [diff] [blame] | 668 | logging.warn('Install step without test step currently not ' |
| 669 | 'supported, installing AND testing instead.') |
Jim Tang | 815b889 | 2018-07-11 12:57:30 +0800 | [diff] [blame] | 670 | steps.append(constants.TEST_STEP) |
yangbill | 848a7d1 | 2018-09-04 19:12:08 +0800 | [diff] [blame] | 671 | tests_exit_code = constants.EXIT_CODE_SUCCESS |
kellyhung | 23c55b8 | 2019-01-04 16:58:14 +0800 | [diff] [blame] | 672 | test_start = time.time() |
Jim Tang | 815b889 | 2018-07-11 12:57:30 +0800 | [diff] [blame] | 673 | if constants.TEST_STEP in steps: |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 674 | if not is_from_test_mapping(test_infos): |
yelinhsieh | 4d5917d | 2019-03-12 17:26:27 +0800 | [diff] [blame] | 675 | tests_exit_code, reporter = test_runner_handler.run_all_tests( |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 676 | results_dir, test_infos, extra_args) |
yelinhsieh | 4d5917d | 2019-03-12 17:26:27 +0800 | [diff] [blame] | 677 | atest_execution_info.AtestExecutionInfo.result_reporters.append(reporter) |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 678 | else: |
| 679 | tests_exit_code = _run_test_mapping_tests( |
| 680 | results_dir, test_infos, extra_args) |
Mike Ma | 0126b9b | 2018-01-11 19:11:16 -0800 | [diff] [blame] | 681 | if args.detect_regression: |
| 682 | regression_args = _get_regression_detection_args(args, results_dir) |
mikehoran | 9b6b44b | 2018-04-09 15:54:58 -0700 | [diff] [blame] | 683 | # TODO(b/110485713): Should not call run_tests here. |
| 684 | reporter = result_reporter.ResultReporter() |
yelinhsieh | 4d5917d | 2019-03-12 17:26:27 +0800 | [diff] [blame] | 685 | atest_execution_info.AtestExecutionInfo.result_reporters.append(reporter) |
Dan Shi | 08c7b72 | 2018-11-29 10:25:59 -0800 | [diff] [blame] | 686 | tests_exit_code |= regression_test_runner.RegressionTestRunner( |
| 687 | '').run_tests( |
| 688 | None, regression_args, reporter) |
kellyhung | 23c55b8 | 2019-01-04 16:58:14 +0800 | [diff] [blame] | 689 | metrics.RunTestsFinishEvent( |
| 690 | duration=metrics_utils.convert_duration(time.time() - test_start)) |
easoncylee | 1cace87 | 2019-09-19 09:03:29 +0800 | [diff] [blame] | 691 | preparation_time = atest_execution_info.preparation_time(test_start) |
| 692 | if preparation_time: |
| 693 | # Send the preparation time only if it's set. |
| 694 | metrics.RunnerFinishEvent( |
| 695 | duration=metrics_utils.convert_duration(preparation_time), |
| 696 | success=True, |
| 697 | runner_name=constants.TF_PREPARATION, |
| 698 | test=[]) |
yangbill | 848a7d1 | 2018-09-04 19:12:08 +0800 | [diff] [blame] | 699 | if tests_exit_code != constants.EXIT_CODE_SUCCESS: |
| 700 | tests_exit_code = constants.EXIT_CODE_TEST_FAILURE |
| 701 | return tests_exit_code |
mikehoran | 63d61b4 | 2017-07-28 15:28:50 -0700 | [diff] [blame] | 702 | |
Simran Basi | 259a2b5 | 2017-06-21 16:14:07 -0700 | [diff] [blame] | 703 | if __name__ == '__main__': |
yelinhsieh | 4d5917d | 2019-03-12 17:26:27 +0800 | [diff] [blame] | 704 | RESULTS_DIR = make_test_run_dir() |
| 705 | with atest_execution_info.AtestExecutionInfo(sys.argv[1:], |
| 706 | RESULTS_DIR) as result_file: |
kellyhung | e3fa175 | 2019-04-23 11:13:41 +0800 | [diff] [blame] | 707 | metrics_base.MetricsBase.tool_name = constants.TOOL_NAME |
yelinhsieh | 4d5917d | 2019-03-12 17:26:27 +0800 | [diff] [blame] | 708 | EXIT_CODE = main(sys.argv[1:], RESULTS_DIR) |
kellyhung | 7d004bb | 2019-04-02 11:54:59 +0800 | [diff] [blame] | 709 | DETECTOR = bug_detector.BugDetector(sys.argv[1:], EXIT_CODE) |
| 710 | metrics.LocalDetectEvent( |
| 711 | detect_type=constants.DETECT_TYPE_BUG_DETECTED, |
| 712 | result=DETECTOR.caught_result) |
yelinhsieh | 4d5917d | 2019-03-12 17:26:27 +0800 | [diff] [blame] | 713 | if result_file: |
| 714 | print('Execution detail has saved in %s' % result_file.name) |
easoncylee | f0fb2b1 | 2019-01-22 15:49:09 +0800 | [diff] [blame] | 715 | sys.exit(EXIT_CODE) |