Shubham Ajmera | fe79349 | 2017-03-16 13:31:35 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 2 | # |
| 3 | # Copyright 2017, The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
| 17 | """ART Run-Test TestRunner |
| 18 | |
| 19 | The testrunner runs the ART run-tests by simply invoking the script. |
| 20 | It fetches the list of eligible tests from art/test directory, and list of |
| 21 | disabled tests from art/test/knownfailures.json. It runs the tests by |
| 22 | invoking art/test/run-test script and checks the exit value to decide if the |
| 23 | test passed or failed. |
| 24 | |
| 25 | Before invoking the script, first build all the tests dependencies. |
| 26 | There are two major build targets for building target and host tests |
| 27 | dependencies: |
| 28 | 1) test-art-host-run-test |
| 29 | 2) test-art-target-run-test |
| 30 | |
| 31 | There are various options to invoke the script which are: |
| 32 | -t: Either the test name as in art/test or the test name including the variant |
| 33 | information. Eg, "-t 001-HelloWorld", |
| 34 | "-t test-art-host-run-test-debug-prebuild-optimizing-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32" |
| 35 | -j: Number of thread workers to be used. Eg - "-j64" |
| 36 | --dry-run: Instead of running the test name, just print its name. |
| 37 | --verbose |
| 38 | -b / --build-dependencies: to build the dependencies before running the test |
| 39 | |
| 40 | To specify any specific variants for the test, use --<<variant-name>>. |
| 41 | For eg, for compiler type as optimizing, use --optimizing. |
| 42 | |
| 43 | |
| 44 | In the end, the script will print the failed and skipped tests if any. |
| 45 | |
| 46 | """ |
Alex Light | 7a1ccf8 | 2017-02-21 09:52:34 -0800 | [diff] [blame] | 47 | import argparse |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 48 | import fnmatch |
| 49 | import itertools |
| 50 | import json |
Shubham Ajmera | 4a5a162 | 2017-03-22 10:07:19 -0700 | [diff] [blame] | 51 | import multiprocessing |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 52 | import os |
| 53 | import re |
| 54 | import subprocess |
| 55 | import sys |
Shubham Ajmera | 29f8968 | 2017-03-24 14:44:10 -0700 | [diff] [blame] | 56 | import tempfile |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 57 | import threading |
| 58 | import time |
| 59 | |
| 60 | import env |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 61 | from target_config import target_config |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 62 | |
| 63 | TARGET_TYPES = set() |
| 64 | RUN_TYPES = set() |
| 65 | PREBUILD_TYPES = set() |
| 66 | COMPILER_TYPES = set() |
| 67 | RELOCATE_TYPES = set() |
| 68 | TRACE_TYPES = set() |
| 69 | GC_TYPES = set() |
| 70 | JNI_TYPES = set() |
| 71 | IMAGE_TYPES = set() |
| 72 | PICTEST_TYPES = set() |
| 73 | DEBUGGABLE_TYPES = set() |
| 74 | ADDRESS_SIZES = set() |
| 75 | OPTIMIZING_COMPILER_TYPES = set() |
Alex Light | 8f2c6d4 | 2017-04-10 16:27:35 -0700 | [diff] [blame] | 76 | JVMTI_TYPES = set() |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 77 | ADDRESS_SIZES_TARGET = {'host': set(), 'target': set()} |
Shubham Ajmera | 186d321 | 2017-07-21 01:20:57 +0000 | [diff] [blame] | 78 | # timeout for individual tests. |
| 79 | # TODO: make it adjustable per tests and for buildbots |
| 80 | timeout = 3000 # 50 minutes |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 81 | |
| 82 | # DISABLED_TEST_CONTAINER holds information about the disabled tests. It is a map |
| 83 | # that has key as the test name (like 001-HelloWorld), and value as set of |
| 84 | # variants that the test is disabled for. |
| 85 | DISABLED_TEST_CONTAINER = {} |
| 86 | |
| 87 | # The Dict contains the list of all possible variants for a given type. For example, |
| 88 | # for key TARGET, the value would be target and host. The list is used to parse |
| 89 | # the test name given as the argument to run. |
| 90 | VARIANT_TYPE_DICT = {} |
| 91 | |
| 92 | # The set contains all the variants of each time. |
| 93 | TOTAL_VARIANTS_SET = set() |
| 94 | |
| 95 | # The colors are used in the output. When a test passes, COLOR_PASS is used, |
| 96 | # and so on. |
| 97 | COLOR_ERROR = '\033[91m' |
| 98 | COLOR_PASS = '\033[92m' |
| 99 | COLOR_SKIP = '\033[93m' |
| 100 | COLOR_NORMAL = '\033[0m' |
| 101 | |
| 102 | # The mutex object is used by the threads for exclusive access of test_count |
| 103 | # to make any changes in its value. |
| 104 | test_count_mutex = threading.Lock() |
Shubham Ajmera | 14d340c | 2017-02-15 18:49:10 +0000 | [diff] [blame] | 105 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 106 | # The set contains the list of all the possible run tests that are in art/test |
| 107 | # directory. |
| 108 | RUN_TEST_SET = set() |
Shubham Ajmera | 14d340c | 2017-02-15 18:49:10 +0000 | [diff] [blame] | 109 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 110 | # The semaphore object is used by the testrunner to limit the number of |
| 111 | # threads to the user requested concurrency value. |
| 112 | semaphore = threading.Semaphore(1) |
Shubham Ajmera | 14d340c | 2017-02-15 18:49:10 +0000 | [diff] [blame] | 113 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 114 | # The mutex object is used to provide exclusive access to a thread to print |
| 115 | # its output. |
| 116 | print_mutex = threading.Lock() |
| 117 | failed_tests = [] |
| 118 | skipped_tests = [] |
| 119 | |
| 120 | # Flags |
Shubham Ajmera | 4a5a162 | 2017-03-22 10:07:19 -0700 | [diff] [blame] | 121 | n_thread = -1 |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 122 | test_count = 0 |
| 123 | total_test_count = 0 |
| 124 | verbose = False |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 125 | dry_run = False |
| 126 | build = False |
| 127 | gdb = False |
| 128 | gdb_arg = '' |
| 129 | stop_testrunner = False |
Shubham Ajmera | 981d99c | 2017-08-17 14:11:08 -0700 | [diff] [blame] | 130 | dex2oat_jobs = -1 # -1 corresponds to default threads for dex2oat |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 131 | |
| 132 | def gather_test_info(): |
| 133 | """The method gathers test information about the test to be run which includes |
| 134 | generating the list of total tests from the art/test directory and the list |
| 135 | of disabled test. It also maps various variants to types. |
| 136 | """ |
| 137 | global TOTAL_VARIANTS_SET |
| 138 | global DISABLED_TEST_CONTAINER |
| 139 | # TODO: Avoid duplication of the variant names in different lists. |
| 140 | VARIANT_TYPE_DICT['pictest'] = {'pictest', 'npictest'} |
| 141 | VARIANT_TYPE_DICT['run'] = {'ndebug', 'debug'} |
| 142 | VARIANT_TYPE_DICT['target'] = {'target', 'host'} |
| 143 | VARIANT_TYPE_DICT['trace'] = {'trace', 'ntrace', 'stream'} |
Richard Uhler | bb00f81 | 2017-02-16 14:21:10 +0000 | [diff] [blame] | 144 | VARIANT_TYPE_DICT['image'] = {'picimage', 'no-image', 'multipicimage'} |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 145 | VARIANT_TYPE_DICT['debuggable'] = {'ndebuggable', 'debuggable'} |
| 146 | VARIANT_TYPE_DICT['gc'] = {'gcstress', 'gcverify', 'cms'} |
| 147 | VARIANT_TYPE_DICT['prebuild'] = {'no-prebuild', 'no-dex2oat', 'prebuild'} |
| 148 | VARIANT_TYPE_DICT['relocate'] = {'relocate-npatchoat', 'relocate', 'no-relocate'} |
| 149 | VARIANT_TYPE_DICT['jni'] = {'jni', 'forcecopy', 'checkjni'} |
| 150 | VARIANT_TYPE_DICT['address_sizes'] = {'64', '32'} |
Alex Light | 43e935d | 2017-06-19 15:40:40 -0700 | [diff] [blame] | 151 | VARIANT_TYPE_DICT['jvmti'] = {'no-jvmti', 'jvmti-stress', 'redefine-stress', 'trace-stress', |
Alex Light | c38c369 | 2017-06-27 15:45:14 -0700 | [diff] [blame] | 152 | 'field-stress', 'step-stress'} |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 153 | VARIANT_TYPE_DICT['compiler'] = {'interp-ac', 'interpreter', 'jit', 'optimizing', |
Jeff Hao | 002b931 | 2017-03-27 16:23:08 -0700 | [diff] [blame] | 154 | 'regalloc_gc', 'speed-profile'} |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 155 | |
| 156 | for v_type in VARIANT_TYPE_DICT: |
| 157 | TOTAL_VARIANTS_SET = TOTAL_VARIANTS_SET.union(VARIANT_TYPE_DICT.get(v_type)) |
| 158 | |
| 159 | test_dir = env.ANDROID_BUILD_TOP + '/art/test' |
| 160 | for f in os.listdir(test_dir): |
| 161 | if fnmatch.fnmatch(f, '[0-9]*'): |
| 162 | RUN_TEST_SET.add(f) |
| 163 | DISABLED_TEST_CONTAINER = get_disabled_test_info() |
| 164 | |
| 165 | |
| 166 | def setup_test_env(): |
| 167 | """The method sets default value for the various variants of the tests if they |
| 168 | are already not set. |
| 169 | """ |
| 170 | if env.ART_TEST_BISECTION: |
| 171 | env.ART_TEST_RUN_TEST_NO_PREBUILD = True |
| 172 | env.ART_TEST_RUN_TEST_PREBUILD = False |
| 173 | # Bisection search writes to standard output. |
| 174 | env.ART_TEST_QUIET = False |
| 175 | |
| 176 | if not TARGET_TYPES: |
| 177 | TARGET_TYPES.add('host') |
| 178 | TARGET_TYPES.add('target') |
| 179 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 180 | if env.ART_TEST_RUN_TEST_NO_PREBUILD: |
| 181 | PREBUILD_TYPES.add('no-prebuild') |
| 182 | if env.ART_TEST_RUN_TEST_NO_DEX2OAT: |
| 183 | PREBUILD_TYPES.add('no-dex2oat') |
Shubham Ajmera | 4b8c53d | 2017-02-15 16:38:16 +0000 | [diff] [blame] | 184 | if env.ART_TEST_RUN_TEST_PREBUILD or not PREBUILD_TYPES: # Default |
| 185 | PREBUILD_TYPES.add('prebuild') |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 186 | |
| 187 | if env.ART_TEST_INTERPRETER_ACCESS_CHECKS: |
| 188 | COMPILER_TYPES.add('interp-ac') |
| 189 | if env.ART_TEST_INTERPRETER: |
| 190 | COMPILER_TYPES.add('interpreter') |
| 191 | if env.ART_TEST_JIT: |
| 192 | COMPILER_TYPES.add('jit') |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 193 | if env.ART_TEST_OPTIMIZING_GRAPH_COLOR: |
| 194 | COMPILER_TYPES.add('regalloc_gc') |
| 195 | OPTIMIZING_COMPILER_TYPES.add('regalloc_gc') |
Nicolas Geoffray | 70b21bd | 2017-03-15 10:18:50 +0000 | [diff] [blame] | 196 | if env.ART_TEST_OPTIMIZING: |
Shubham Ajmera | 4b8c53d | 2017-02-15 16:38:16 +0000 | [diff] [blame] | 197 | COMPILER_TYPES.add('optimizing') |
| 198 | OPTIMIZING_COMPILER_TYPES.add('optimizing') |
Jeff Hao | 002b931 | 2017-03-27 16:23:08 -0700 | [diff] [blame] | 199 | if env.ART_TEST_SPEED_PROFILE: |
| 200 | COMPILER_TYPES.add('speed-profile') |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 201 | |
Alex Light | 8f2c6d4 | 2017-04-10 16:27:35 -0700 | [diff] [blame] | 202 | # By default only run without jvmti |
| 203 | if not JVMTI_TYPES: |
| 204 | JVMTI_TYPES.add('no-jvmti') |
| 205 | |
Nicolas Geoffray | 70b21bd | 2017-03-15 10:18:50 +0000 | [diff] [blame] | 206 | # By default we run all 'compiler' variants. |
| 207 | if not COMPILER_TYPES: |
| 208 | COMPILER_TYPES.add('optimizing') |
| 209 | COMPILER_TYPES.add('jit') |
| 210 | COMPILER_TYPES.add('interpreter') |
| 211 | COMPILER_TYPES.add('interp-ac') |
Jeff Hao | 002b931 | 2017-03-27 16:23:08 -0700 | [diff] [blame] | 212 | COMPILER_TYPES.add('speed-profile') |
Nicolas Geoffray | 70b21bd | 2017-03-15 10:18:50 +0000 | [diff] [blame] | 213 | OPTIMIZING_COMPILER_TYPES.add('optimizing') |
| 214 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 215 | if env.ART_TEST_RUN_TEST_RELOCATE: |
| 216 | RELOCATE_TYPES.add('relocate') |
| 217 | if env.ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT: |
| 218 | RELOCATE_TYPES.add('relocate-npatchoat') |
Shubham Ajmera | 4b8c53d | 2017-02-15 16:38:16 +0000 | [diff] [blame] | 219 | if not RELOCATE_TYPES: # Default |
| 220 | RELOCATE_TYPES.add('no-relocate') |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 221 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 222 | if env.ART_TEST_TRACE: |
| 223 | TRACE_TYPES.add('trace') |
| 224 | if env.ART_TEST_TRACE_STREAM: |
| 225 | TRACE_TYPES.add('stream') |
Shubham Ajmera | 4b8c53d | 2017-02-15 16:38:16 +0000 | [diff] [blame] | 226 | if not TRACE_TYPES: # Default |
| 227 | TRACE_TYPES.add('ntrace') |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 228 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 229 | if env.ART_TEST_GC_STRESS: |
| 230 | GC_TYPES.add('gcstress') |
| 231 | if env.ART_TEST_GC_VERIFY: |
| 232 | GC_TYPES.add('gcverify') |
Shubham Ajmera | 4b8c53d | 2017-02-15 16:38:16 +0000 | [diff] [blame] | 233 | if not GC_TYPES: # Default |
| 234 | GC_TYPES.add('cms') |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 235 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 236 | if env.ART_TEST_JNI_FORCECOPY: |
| 237 | JNI_TYPES.add('forcecopy') |
Shubham Ajmera | 4b8c53d | 2017-02-15 16:38:16 +0000 | [diff] [blame] | 238 | if not JNI_TYPES: # Default |
| 239 | JNI_TYPES.add('checkjni') |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 240 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 241 | if env.ART_TEST_RUN_TEST_NO_IMAGE: |
| 242 | IMAGE_TYPES.add('no-image') |
| 243 | if env.ART_TEST_RUN_TEST_MULTI_IMAGE: |
| 244 | IMAGE_TYPES.add('multipicimage') |
Shubham Ajmera | 4b8c53d | 2017-02-15 16:38:16 +0000 | [diff] [blame] | 245 | if env.ART_TEST_RUN_TEST_IMAGE or not IMAGE_TYPES: # Default |
| 246 | IMAGE_TYPES.add('picimage') |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 247 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 248 | if env.ART_TEST_PIC_TEST: |
| 249 | PICTEST_TYPES.add('pictest') |
Shubham Ajmera | 4b8c53d | 2017-02-15 16:38:16 +0000 | [diff] [blame] | 250 | if not PICTEST_TYPES: # Default |
| 251 | PICTEST_TYPES.add('npictest') |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 252 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 253 | if env.ART_TEST_RUN_TEST_NDEBUG: |
| 254 | RUN_TYPES.add('ndebug') |
Shubham Ajmera | 4b8c53d | 2017-02-15 16:38:16 +0000 | [diff] [blame] | 255 | if env.ART_TEST_RUN_TEST_DEBUG or not RUN_TYPES: # Default |
| 256 | RUN_TYPES.add('debug') |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 257 | |
| 258 | if env.ART_TEST_RUN_TEST_DEBUGGABLE: |
| 259 | DEBUGGABLE_TYPES.add('debuggable') |
Shubham Ajmera | 4b8c53d | 2017-02-15 16:38:16 +0000 | [diff] [blame] | 260 | if not DEBUGGABLE_TYPES: # Default |
| 261 | DEBUGGABLE_TYPES.add('ndebuggable') |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 262 | |
| 263 | if not ADDRESS_SIZES: |
| 264 | ADDRESS_SIZES_TARGET['target'].add(env.ART_PHONY_TEST_TARGET_SUFFIX) |
| 265 | ADDRESS_SIZES_TARGET['host'].add(env.ART_PHONY_TEST_HOST_SUFFIX) |
| 266 | if env.ART_TEST_RUN_TEST_2ND_ARCH: |
Igor Murashkin | 6b61c80 | 2017-04-03 14:33:22 -0700 | [diff] [blame] | 267 | ADDRESS_SIZES_TARGET['host'].add(env.ART_2ND_PHONY_TEST_HOST_SUFFIX) |
| 268 | ADDRESS_SIZES_TARGET['target'].add(env.ART_2ND_PHONY_TEST_TARGET_SUFFIX) |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 269 | else: |
| 270 | ADDRESS_SIZES_TARGET['host'] = ADDRESS_SIZES_TARGET['host'].union(ADDRESS_SIZES) |
| 271 | ADDRESS_SIZES_TARGET['target'] = ADDRESS_SIZES_TARGET['target'].union(ADDRESS_SIZES) |
| 272 | |
Shubham Ajmera | 4a5a162 | 2017-03-22 10:07:19 -0700 | [diff] [blame] | 273 | global n_thread |
| 274 | if n_thread is -1: |
| 275 | if 'target' in TARGET_TYPES: |
| 276 | n_thread = get_default_threads('target') |
| 277 | else: |
| 278 | n_thread = get_default_threads('host') |
| 279 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 280 | global semaphore |
| 281 | semaphore = threading.Semaphore(n_thread) |
| 282 | |
Shubham Ajmera | 22499e2 | 2017-03-22 18:33:37 -0700 | [diff] [blame] | 283 | if not sys.stdout.isatty(): |
| 284 | global COLOR_ERROR |
| 285 | global COLOR_PASS |
| 286 | global COLOR_SKIP |
| 287 | global COLOR_NORMAL |
| 288 | COLOR_ERROR = '' |
| 289 | COLOR_PASS = '' |
| 290 | COLOR_SKIP = '' |
| 291 | COLOR_NORMAL = '' |
| 292 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 293 | |
| 294 | def run_tests(tests): |
| 295 | """Creates thread workers to run the tests. |
| 296 | |
| 297 | The method generates command and thread worker to run the tests. Depending on |
| 298 | the user input for the number of threads to be used, the method uses a |
| 299 | semaphore object to keep a count in control for the thread workers. When a new |
| 300 | worker is created, it acquires the semaphore object, and when the number of |
| 301 | workers reaches the maximum allowed concurrency, the method wait for an |
| 302 | existing thread worker to release the semaphore object. Worker releases the |
| 303 | semaphore object when they finish printing the output. |
| 304 | |
| 305 | Args: |
| 306 | tests: The set of tests to be run. |
| 307 | """ |
| 308 | options_all = '' |
| 309 | global total_test_count |
| 310 | total_test_count = len(tests) |
| 311 | total_test_count *= len(RUN_TYPES) |
| 312 | total_test_count *= len(PREBUILD_TYPES) |
| 313 | total_test_count *= len(RELOCATE_TYPES) |
| 314 | total_test_count *= len(TRACE_TYPES) |
| 315 | total_test_count *= len(GC_TYPES) |
| 316 | total_test_count *= len(JNI_TYPES) |
| 317 | total_test_count *= len(IMAGE_TYPES) |
| 318 | total_test_count *= len(PICTEST_TYPES) |
| 319 | total_test_count *= len(DEBUGGABLE_TYPES) |
| 320 | total_test_count *= len(COMPILER_TYPES) |
Alex Light | 8f2c6d4 | 2017-04-10 16:27:35 -0700 | [diff] [blame] | 321 | total_test_count *= len(JVMTI_TYPES) |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 322 | target_address_combinations = 0 |
| 323 | for target in TARGET_TYPES: |
| 324 | for address_size in ADDRESS_SIZES_TARGET[target]: |
| 325 | target_address_combinations += 1 |
| 326 | total_test_count *= target_address_combinations |
| 327 | |
| 328 | if env.ART_TEST_WITH_STRACE: |
| 329 | options_all += ' --strace' |
| 330 | |
| 331 | if env.ART_TEST_RUN_TEST_ALWAYS_CLEAN: |
| 332 | options_all += ' --always-clean' |
| 333 | |
| 334 | if env.ART_TEST_BISECTION: |
| 335 | options_all += ' --bisection-search' |
| 336 | |
| 337 | if env.ART_TEST_ANDROID_ROOT: |
| 338 | options_all += ' --android-root ' + env.ART_TEST_ANDROID_ROOT |
| 339 | |
| 340 | if gdb: |
| 341 | options_all += ' --gdb' |
| 342 | if gdb_arg: |
| 343 | options_all += ' --gdb-arg ' + gdb_arg |
| 344 | |
Shubham Ajmera | 981d99c | 2017-08-17 14:11:08 -0700 | [diff] [blame] | 345 | if dex2oat_jobs != -1: |
| 346 | options_all += ' --dex2oat-jobs ' + str(dex2oat_jobs) |
| 347 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 348 | config = itertools.product(tests, TARGET_TYPES, RUN_TYPES, PREBUILD_TYPES, |
| 349 | COMPILER_TYPES, RELOCATE_TYPES, TRACE_TYPES, |
| 350 | GC_TYPES, JNI_TYPES, IMAGE_TYPES, PICTEST_TYPES, |
Alex Light | 8f2c6d4 | 2017-04-10 16:27:35 -0700 | [diff] [blame] | 351 | DEBUGGABLE_TYPES, JVMTI_TYPES) |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 352 | |
| 353 | for test, target, run, prebuild, compiler, relocate, trace, gc, \ |
Alex Light | 8f2c6d4 | 2017-04-10 16:27:35 -0700 | [diff] [blame] | 354 | jni, image, pictest, debuggable, jvmti in config: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 355 | for address_size in ADDRESS_SIZES_TARGET[target]: |
| 356 | if stop_testrunner: |
| 357 | # When ART_TEST_KEEP_GOING is set to false, then as soon as a test |
| 358 | # fails, stop_testrunner is set to True. When this happens, the method |
| 359 | # stops creating any any thread and wait for all the exising threads |
| 360 | # to end. |
| 361 | while threading.active_count() > 2: |
| 362 | time.sleep(0.1) |
| 363 | return |
| 364 | test_name = 'test-art-' |
| 365 | test_name += target + '-run-test-' |
| 366 | test_name += run + '-' |
| 367 | test_name += prebuild + '-' |
| 368 | test_name += compiler + '-' |
| 369 | test_name += relocate + '-' |
| 370 | test_name += trace + '-' |
| 371 | test_name += gc + '-' |
| 372 | test_name += jni + '-' |
| 373 | test_name += image + '-' |
| 374 | test_name += pictest + '-' |
| 375 | test_name += debuggable + '-' |
Alex Light | 8f2c6d4 | 2017-04-10 16:27:35 -0700 | [diff] [blame] | 376 | test_name += jvmti + '-' |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 377 | test_name += test |
| 378 | test_name += address_size |
| 379 | |
| 380 | variant_set = {target, run, prebuild, compiler, relocate, trace, gc, jni, |
Alex Light | 8f2c6d4 | 2017-04-10 16:27:35 -0700 | [diff] [blame] | 381 | image, pictest, debuggable, jvmti, address_size} |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 382 | |
| 383 | options_test = options_all |
| 384 | |
| 385 | if target == 'host': |
| 386 | options_test += ' --host' |
| 387 | |
| 388 | if run == 'ndebug': |
| 389 | options_test += ' -O' |
| 390 | |
| 391 | if prebuild == 'prebuild': |
| 392 | options_test += ' --prebuild' |
| 393 | elif prebuild == 'no-prebuild': |
| 394 | options_test += ' --no-prebuild' |
| 395 | elif prebuild == 'no-dex2oat': |
| 396 | options_test += ' --no-prebuild --no-dex2oat' |
| 397 | |
| 398 | if compiler == 'optimizing': |
| 399 | options_test += ' --optimizing' |
| 400 | elif compiler == 'regalloc_gc': |
| 401 | options_test += ' --optimizing -Xcompiler-option --register-allocation-strategy=graph-color' |
| 402 | elif compiler == 'interpreter': |
| 403 | options_test += ' --interpreter' |
| 404 | elif compiler == 'interp-ac': |
| 405 | options_test += ' --interpreter --verify-soft-fail' |
| 406 | elif compiler == 'jit': |
| 407 | options_test += ' --jit' |
Jeff Hao | 002b931 | 2017-03-27 16:23:08 -0700 | [diff] [blame] | 408 | elif compiler == 'speed-profile': |
| 409 | options_test += ' --random-profile' |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 410 | |
| 411 | if relocate == 'relocate': |
| 412 | options_test += ' --relocate' |
| 413 | elif relocate == 'no-relocate': |
| 414 | options_test += ' --no-relocate' |
| 415 | elif relocate == 'relocate-npatchoat': |
| 416 | options_test += ' --relocate --no-patchoat' |
| 417 | |
| 418 | if trace == 'trace': |
| 419 | options_test += ' --trace' |
| 420 | elif trace == 'stream': |
| 421 | options_test += ' --trace --stream' |
| 422 | |
| 423 | if gc == 'gcverify': |
| 424 | options_test += ' --gcverify' |
| 425 | elif gc == 'gcstress': |
| 426 | options_test += ' --gcstress' |
| 427 | |
| 428 | if jni == 'forcecopy': |
| 429 | options_test += ' --runtime-option -Xjniopts:forcecopy' |
| 430 | elif jni == 'checkjni': |
| 431 | options_test += ' --runtime-option -Xcheck:jni' |
| 432 | |
| 433 | if image == 'no-image': |
| 434 | options_test += ' --no-image' |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 435 | elif image == 'multipicimage': |
| 436 | options_test += ' --multi-image' |
| 437 | |
| 438 | if pictest == 'pictest': |
| 439 | options_test += ' --pic-test' |
| 440 | |
| 441 | if debuggable == 'debuggable': |
| 442 | options_test += ' --debuggable' |
| 443 | |
Alex Light | 8f2c6d4 | 2017-04-10 16:27:35 -0700 | [diff] [blame] | 444 | if jvmti == 'jvmti-stress': |
Alex Light | 43e935d | 2017-06-19 15:40:40 -0700 | [diff] [blame] | 445 | options_test += ' --jvmti-trace-stress --jvmti-redefine-stress --jvmti-field-stress' |
| 446 | elif jvmti == 'field-stress': |
| 447 | options_test += ' --jvmti-field-stress' |
Alex Light | b7edcda | 2017-04-27 13:20:31 -0700 | [diff] [blame] | 448 | elif jvmti == 'trace-stress': |
| 449 | options_test += ' --jvmti-trace-stress' |
| 450 | elif jvmti == 'redefine-stress': |
| 451 | options_test += ' --jvmti-redefine-stress' |
Alex Light | c38c369 | 2017-06-27 15:45:14 -0700 | [diff] [blame] | 452 | elif jvmti == 'step-stress': |
| 453 | options_test += ' --jvmti-step-stress' |
Alex Light | 8f2c6d4 | 2017-04-10 16:27:35 -0700 | [diff] [blame] | 454 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 455 | if address_size == '64': |
| 456 | options_test += ' --64' |
| 457 | |
| 458 | if env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES: |
| 459 | options_test += ' --instruction-set-features' + env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES |
| 460 | |
| 461 | elif address_size == '32': |
| 462 | if env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES: |
| 463 | options_test += ' --instruction-set-features ' + \ |
| 464 | env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES |
| 465 | |
Igor Murashkin | 8889a89 | 2017-04-24 16:09:15 -0700 | [diff] [blame] | 466 | # Use the default run-test behavior unless ANDROID_COMPILE_WITH_JACK is explicitly set. |
| 467 | if env.ANDROID_COMPILE_WITH_JACK == True: |
| 468 | options_test += ' --build-with-jack' |
| 469 | elif env.ANDROID_COMPILE_WITH_JACK == False: |
| 470 | options_test += ' --build-with-javac-dx' |
| 471 | |
Shubham Ajmera | 29f8968 | 2017-03-24 14:44:10 -0700 | [diff] [blame] | 472 | # TODO(http://36039166): This is a temporary solution to |
| 473 | # fix build breakages. |
| 474 | options_test = (' --output-path %s') % ( |
| 475 | tempfile.mkdtemp(dir=env.ART_HOST_TEST_DIR)) + options_test |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 476 | |
| 477 | run_test_sh = env.ANDROID_BUILD_TOP + '/art/test/run-test' |
| 478 | command = run_test_sh + ' ' + options_test + ' ' + test |
| 479 | |
| 480 | semaphore.acquire() |
| 481 | worker = threading.Thread(target=run_test, args=(command, test, variant_set, test_name)) |
| 482 | worker.daemon = True |
| 483 | worker.start() |
| 484 | |
| 485 | while threading.active_count() > 2: |
| 486 | time.sleep(0.1) |
| 487 | |
| 488 | |
| 489 | def run_test(command, test, test_variant, test_name): |
| 490 | """Runs the test. |
| 491 | |
| 492 | It invokes art/test/run-test script to run the test. The output of the script |
| 493 | is checked, and if it ends with "Succeeded!", it assumes that the tests |
| 494 | passed, otherwise, put it in the list of failed test. Before actually running |
| 495 | the test, it also checks if the test is placed in the list of disabled tests, |
| 496 | and if yes, it skips running it, and adds the test in the list of skipped |
| 497 | tests. The method uses print_text method to actually print the output. After |
| 498 | successfully running and capturing the output for the test, it releases the |
| 499 | semaphore object. |
| 500 | |
| 501 | Args: |
| 502 | command: The command to be used to invoke the script |
| 503 | test: The name of the test without the variant information. |
| 504 | test_variant: The set of variant for the test. |
| 505 | test_name: The name of the test along with the variants. |
| 506 | """ |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 507 | global stop_testrunner |
Shubham Ajmera | bbd8433 | 2017-02-17 00:41:10 +0000 | [diff] [blame] | 508 | try: |
| 509 | if is_test_disabled(test, test_variant): |
| 510 | test_skipped = True |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 511 | else: |
Shubham Ajmera | bbd8433 | 2017-02-17 00:41:10 +0000 | [diff] [blame] | 512 | test_skipped = False |
Shubham Ajmera | b4949f5 | 2017-05-08 13:52:46 -0700 | [diff] [blame] | 513 | if gdb: |
| 514 | proc = subprocess.Popen(command.split(), stderr=subprocess.STDOUT, universal_newlines=True) |
| 515 | else: |
| 516 | proc = subprocess.Popen(command.split(), stderr=subprocess.STDOUT, stdout = subprocess.PIPE, |
| 517 | universal_newlines=True) |
Shubham Ajmera | 186d321 | 2017-07-21 01:20:57 +0000 | [diff] [blame] | 518 | script_output = proc.communicate(timeout=timeout)[0] |
Shubham Ajmera | bbd8433 | 2017-02-17 00:41:10 +0000 | [diff] [blame] | 519 | test_passed = not proc.wait() |
| 520 | |
| 521 | if not test_skipped: |
| 522 | if test_passed: |
| 523 | print_test_info(test_name, 'PASS') |
| 524 | else: |
Shubham Ajmera | 3092c6e | 2017-03-24 16:19:48 -0700 | [diff] [blame] | 525 | failed_tests.append((test_name, script_output)) |
Shubham Ajmera | bbd8433 | 2017-02-17 00:41:10 +0000 | [diff] [blame] | 526 | if not env.ART_TEST_KEEP_GOING: |
| 527 | stop_testrunner = True |
| 528 | print_test_info(test_name, 'FAIL', ('%s\n%s') % ( |
| 529 | command, script_output)) |
| 530 | elif not dry_run: |
| 531 | print_test_info(test_name, 'SKIP') |
| 532 | skipped_tests.append(test_name) |
| 533 | else: |
| 534 | print_test_info(test_name, '') |
Shubham Ajmera | fe79349 | 2017-03-16 13:31:35 -0700 | [diff] [blame] | 535 | except subprocess.TimeoutExpired as e: |
Shubham Ajmera | 186d321 | 2017-07-21 01:20:57 +0000 | [diff] [blame] | 536 | failed_tests.append((test_name, 'Timed out in %d seconds' % timeout)) |
Shubham Ajmera | 3092c6e | 2017-03-24 16:19:48 -0700 | [diff] [blame] | 537 | print_test_info(test_name, 'TIMEOUT', 'Timed out in %d seconds\n%s' % ( |
Shubham Ajmera | 186d321 | 2017-07-21 01:20:57 +0000 | [diff] [blame] | 538 | timeout, command)) |
Shubham Ajmera | fe79349 | 2017-03-16 13:31:35 -0700 | [diff] [blame] | 539 | except Exception as e: |
Shubham Ajmera | 3092c6e | 2017-03-24 16:19:48 -0700 | [diff] [blame] | 540 | failed_tests.append((test_name, str(e))) |
| 541 | print_test_info(test_name, 'FAIL', |
| 542 | ('%s\n%s\n\n') % (command, str(e))) |
Shubham Ajmera | bbd8433 | 2017-02-17 00:41:10 +0000 | [diff] [blame] | 543 | finally: |
| 544 | semaphore.release() |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 545 | |
| 546 | |
Shubham Ajmera | 14d340c | 2017-02-15 18:49:10 +0000 | [diff] [blame] | 547 | def print_test_info(test_name, result, failed_test_info=""): |
| 548 | """Print the continous test information |
| 549 | |
| 550 | If verbose is set to True, it continuously prints test status information |
| 551 | on a new line. |
| 552 | If verbose is set to False, it keeps on erasing test |
| 553 | information by overriding it with the latest test information. Also, |
| 554 | in this case it stictly makes sure that the information length doesn't |
| 555 | exceed the console width. It does so by shortening the test_name. |
| 556 | |
| 557 | When a test fails, it prints the output of the run-test script and |
| 558 | command used to invoke the script. It doesn't override the failing |
| 559 | test information in either of the cases. |
| 560 | """ |
Shubham Ajmera | bbd8433 | 2017-02-17 00:41:10 +0000 | [diff] [blame] | 561 | |
Shubham Ajmera | 14d340c | 2017-02-15 18:49:10 +0000 | [diff] [blame] | 562 | global test_count |
| 563 | info = '' |
| 564 | if not verbose: |
| 565 | # Without --verbose, the testrunner erases passing test info. It |
| 566 | # does that by overriding the printed text with white spaces all across |
| 567 | # the console width. |
| 568 | console_width = int(os.popen('stty size', 'r').read().split()[1]) |
| 569 | info = '\r' + ' ' * console_width + '\r' |
Shubham Ajmera | bbd8433 | 2017-02-17 00:41:10 +0000 | [diff] [blame] | 570 | try: |
| 571 | print_mutex.acquire() |
| 572 | test_count += 1 |
| 573 | percent = (test_count * 100) / total_test_count |
| 574 | progress_info = ('[ %d%% %d/%d ]') % ( |
| 575 | percent, |
| 576 | test_count, |
| 577 | total_test_count) |
Shubham Ajmera | 14d340c | 2017-02-15 18:49:10 +0000 | [diff] [blame] | 578 | |
Shubham Ajmera | fe79349 | 2017-03-16 13:31:35 -0700 | [diff] [blame] | 579 | if result == 'FAIL' or result == 'TIMEOUT': |
Shubham Ajmera | bbd8433 | 2017-02-17 00:41:10 +0000 | [diff] [blame] | 580 | info += ('%s %s %s\n%s\n') % ( |
Shubham Ajmera | 14d340c | 2017-02-15 18:49:10 +0000 | [diff] [blame] | 581 | progress_info, |
| 582 | test_name, |
Shubham Ajmera | fe79349 | 2017-03-16 13:31:35 -0700 | [diff] [blame] | 583 | COLOR_ERROR + result + COLOR_NORMAL, |
Shubham Ajmera | bbd8433 | 2017-02-17 00:41:10 +0000 | [diff] [blame] | 584 | failed_test_info) |
| 585 | else: |
| 586 | result_text = '' |
| 587 | if result == 'PASS': |
| 588 | result_text += COLOR_PASS + 'PASS' + COLOR_NORMAL |
| 589 | elif result == 'SKIP': |
| 590 | result_text += COLOR_SKIP + 'SKIP' + COLOR_NORMAL |
| 591 | |
| 592 | if verbose: |
| 593 | info += ('%s %s %s\n') % ( |
| 594 | progress_info, |
| 595 | test_name, |
| 596 | result_text) |
| 597 | else: |
| 598 | total_output_length = 2 # Two spaces |
| 599 | total_output_length += len(progress_info) |
| 600 | total_output_length += len(result) |
| 601 | allowed_test_length = console_width - total_output_length |
| 602 | test_name_len = len(test_name) |
| 603 | if allowed_test_length < test_name_len: |
Shubham Ajmera | fe79349 | 2017-03-16 13:31:35 -0700 | [diff] [blame] | 604 | test_name = ('...%s') % ( |
| 605 | test_name[-(allowed_test_length - 3):]) |
Alex Light | c14311c | 2017-02-23 17:02:46 -0800 | [diff] [blame] | 606 | info += ('%s %s %s') % ( |
| 607 | progress_info, |
| 608 | test_name, |
| 609 | result_text) |
Shubham Ajmera | bbd8433 | 2017-02-17 00:41:10 +0000 | [diff] [blame] | 610 | print_text(info) |
Shubham Ajmera | fe79349 | 2017-03-16 13:31:35 -0700 | [diff] [blame] | 611 | except Exception as e: |
Shubham Ajmera | bbd8433 | 2017-02-17 00:41:10 +0000 | [diff] [blame] | 612 | print_text(('%s\n%s\n') % (test_name, str(e))) |
| 613 | failed_tests.append(test_name) |
| 614 | finally: |
| 615 | print_mutex.release() |
Shubham Ajmera | 14d340c | 2017-02-15 18:49:10 +0000 | [diff] [blame] | 616 | |
Shubham Ajmera | 14de5c4 | 2017-03-13 10:51:14 -0700 | [diff] [blame] | 617 | def verify_knownfailure_entry(entry): |
| 618 | supported_field = { |
Shubham Ajmera | fe79349 | 2017-03-16 13:31:35 -0700 | [diff] [blame] | 619 | 'tests' : (list, str), |
Alex Light | 6fdc1b6 | 2017-09-18 11:33:56 -0700 | [diff] [blame] | 620 | 'test_patterns' : (list,), |
Shubham Ajmera | fe79349 | 2017-03-16 13:31:35 -0700 | [diff] [blame] | 621 | 'description' : (list, str), |
| 622 | 'bug' : (str,), |
| 623 | 'variant' : (str,), |
Shubham Ajmera | 14de5c4 | 2017-03-13 10:51:14 -0700 | [diff] [blame] | 624 | 'env_vars' : (dict,), |
| 625 | } |
| 626 | for field in entry: |
| 627 | field_type = type(entry[field]) |
| 628 | if field_type not in supported_field[field]: |
| 629 | raise ValueError('%s is not supported type for %s\n%s' % ( |
| 630 | str(field_type), |
| 631 | field, |
| 632 | str(entry))) |
| 633 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 634 | def get_disabled_test_info(): |
| 635 | """Generate set of known failures. |
| 636 | |
| 637 | It parses the art/test/knownfailures.json file to generate the list of |
| 638 | disabled tests. |
| 639 | |
| 640 | Returns: |
| 641 | The method returns a dict of tests mapped to the variants list |
| 642 | for which the test should not be run. |
| 643 | """ |
| 644 | known_failures_file = env.ANDROID_BUILD_TOP + '/art/test/knownfailures.json' |
| 645 | with open(known_failures_file) as known_failures_json: |
| 646 | known_failures_info = json.loads(known_failures_json.read()) |
| 647 | |
| 648 | disabled_test_info = {} |
| 649 | for failure in known_failures_info: |
Shubham Ajmera | 14de5c4 | 2017-03-13 10:51:14 -0700 | [diff] [blame] | 650 | verify_knownfailure_entry(failure) |
| 651 | tests = failure.get('tests', []) |
Shubham Ajmera | fe79349 | 2017-03-16 13:31:35 -0700 | [diff] [blame] | 652 | if isinstance(tests, str): |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 653 | tests = [tests] |
Alex Light | 6fdc1b6 | 2017-09-18 11:33:56 -0700 | [diff] [blame] | 654 | patterns = failure.get("test_patterns", []) |
| 655 | if (not isinstance(patterns, list)): |
| 656 | raise ValueError("test_patters is not a list in %s" % failure) |
| 657 | |
| 658 | tests += [f for f in RUN_TEST_SET if any(re.match(pat, f) is not None for pat in patterns)] |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 659 | variants = parse_variants(failure.get('variant')) |
| 660 | env_vars = failure.get('env_vars') |
Shubham Ajmera | 14de5c4 | 2017-03-13 10:51:14 -0700 | [diff] [blame] | 661 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 662 | if check_env_vars(env_vars): |
| 663 | for test in tests: |
Shubham Ajmera | 14de5c4 | 2017-03-13 10:51:14 -0700 | [diff] [blame] | 664 | if test not in RUN_TEST_SET: |
| 665 | raise ValueError('%s is not a valid run-test' % ( |
| 666 | test)) |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 667 | if test in disabled_test_info: |
| 668 | disabled_test_info[test] = disabled_test_info[test].union(variants) |
| 669 | else: |
| 670 | disabled_test_info[test] = variants |
| 671 | return disabled_test_info |
| 672 | |
| 673 | |
| 674 | def check_env_vars(env_vars): |
| 675 | """Checks if the env variables are set as required to run the test. |
| 676 | |
| 677 | Returns: |
| 678 | True if all the env variables are set as required, otherwise False. |
| 679 | """ |
| 680 | |
| 681 | if not env_vars: |
| 682 | return True |
| 683 | for key in env_vars: |
| 684 | if env.get_env(key) != env_vars.get(key): |
| 685 | return False |
| 686 | return True |
| 687 | |
| 688 | |
| 689 | def is_test_disabled(test, variant_set): |
| 690 | """Checks if the test along with the variant_set is disabled. |
| 691 | |
| 692 | Args: |
| 693 | test: The name of the test as in art/test directory. |
| 694 | variant_set: Variants to be used for the test. |
| 695 | Returns: |
| 696 | True, if the test is disabled. |
| 697 | """ |
| 698 | if dry_run: |
| 699 | return True |
Alex Light | bc319b2 | 2017-02-17 14:21:33 -0800 | [diff] [blame] | 700 | if test in env.EXTRA_DISABLED_TESTS: |
| 701 | return True |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 702 | variants_list = DISABLED_TEST_CONTAINER.get(test, {}) |
| 703 | for variants in variants_list: |
| 704 | variants_present = True |
| 705 | for variant in variants: |
| 706 | if variant not in variant_set: |
| 707 | variants_present = False |
| 708 | break |
| 709 | if variants_present: |
| 710 | return True |
| 711 | return False |
| 712 | |
| 713 | |
| 714 | def parse_variants(variants): |
| 715 | """Parse variants fetched from art/test/knownfailures.json. |
| 716 | """ |
| 717 | if not variants: |
| 718 | variants = '' |
| 719 | for variant in TOTAL_VARIANTS_SET: |
| 720 | variants += variant |
| 721 | variants += '|' |
| 722 | variants = variants[:-1] |
| 723 | variant_list = set() |
| 724 | or_variants = variants.split('|') |
| 725 | for or_variant in or_variants: |
| 726 | and_variants = or_variant.split('&') |
| 727 | variant = set() |
| 728 | for and_variant in and_variants: |
| 729 | and_variant = and_variant.strip() |
Shubham Ajmera | 14de5c4 | 2017-03-13 10:51:14 -0700 | [diff] [blame] | 730 | if and_variant not in TOTAL_VARIANTS_SET: |
| 731 | raise ValueError('%s is not a valid variant' % ( |
| 732 | and_variant)) |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 733 | variant.add(and_variant) |
| 734 | variant_list.add(frozenset(variant)) |
| 735 | return variant_list |
| 736 | |
| 737 | def print_text(output): |
| 738 | sys.stdout.write(output) |
| 739 | sys.stdout.flush() |
| 740 | |
| 741 | def print_analysis(): |
| 742 | if not verbose: |
Shubham Ajmera | 14d340c | 2017-02-15 18:49:10 +0000 | [diff] [blame] | 743 | # Without --verbose, the testrunner erases passing test info. It |
| 744 | # does that by overriding the printed text with white spaces all across |
| 745 | # the console width. |
| 746 | console_width = int(os.popen('stty size', 'r').read().split()[1]) |
| 747 | eraser_text = '\r' + ' ' * console_width + '\r' |
| 748 | print_text(eraser_text) |
Shubham Ajmera | cbf5628 | 2017-03-13 09:54:23 -0700 | [diff] [blame] | 749 | |
| 750 | # Prints information about the total tests run. |
| 751 | # E.g., "2/38 (5%) tests passed". |
| 752 | passed_test_count = total_test_count - len(skipped_tests) - len(failed_tests) |
| 753 | passed_test_information = ('%d/%d (%d%%) %s passed.\n') % ( |
| 754 | passed_test_count, |
| 755 | total_test_count, |
| 756 | (passed_test_count*100)/total_test_count, |
| 757 | 'tests' if passed_test_count > 1 else 'test') |
| 758 | print_text(passed_test_information) |
| 759 | |
| 760 | # Prints the list of skipped tests, if any. |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 761 | if skipped_tests: |
Shubham Ajmera | 3092c6e | 2017-03-24 16:19:48 -0700 | [diff] [blame] | 762 | print_text(COLOR_SKIP + 'SKIPPED TESTS: ' + COLOR_NORMAL + '\n') |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 763 | for test in skipped_tests: |
| 764 | print_text(test + '\n') |
| 765 | print_text('\n') |
| 766 | |
Shubham Ajmera | cbf5628 | 2017-03-13 09:54:23 -0700 | [diff] [blame] | 767 | # Prints the list of failed tests, if any. |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 768 | if failed_tests: |
Shubham Ajmera | 3092c6e | 2017-03-24 16:19:48 -0700 | [diff] [blame] | 769 | print_text(COLOR_ERROR + 'FAILED: ' + COLOR_NORMAL + '\n') |
| 770 | for test_info in failed_tests: |
| 771 | print_text(('%s\n%s\n' % (test_info[0], test_info[1]))) |
Andreas Gampe | 0dd7e85 | 2017-05-24 21:44:23 -0700 | [diff] [blame] | 772 | print_text(COLOR_ERROR + '----------' + COLOR_NORMAL + '\n') |
| 773 | for failed_test in sorted([test_info[0] for test_info in failed_tests]): |
| 774 | print_text(('%s\n' % (failed_test))) |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 775 | |
| 776 | |
| 777 | def parse_test_name(test_name): |
| 778 | """Parses the testname provided by the user. |
| 779 | It supports two types of test_name: |
| 780 | 1) Like 001-HelloWorld. In this case, it will just verify if the test actually |
| 781 | exists and if it does, it returns the testname. |
| 782 | 2) Like test-art-host-run-test-debug-prebuild-interpreter-no-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32 |
| 783 | In this case, it will parse all the variants and check if they are placed |
| 784 | correctly. If yes, it will set the various VARIANT_TYPES to use the |
| 785 | variants required to run the test. Again, it returns the test_name |
| 786 | without the variant information like 001-HelloWorld. |
| 787 | """ |
Shubham Ajmera | faf1250 | 2017-02-15 17:19:44 +0000 | [diff] [blame] | 788 | test_set = set() |
| 789 | for test in RUN_TEST_SET: |
| 790 | if test.startswith(test_name): |
| 791 | test_set.add(test) |
| 792 | if test_set: |
| 793 | return test_set |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 794 | |
| 795 | regex = '^test-art-' |
| 796 | regex += '(' + '|'.join(VARIANT_TYPE_DICT['target']) + ')-' |
| 797 | regex += 'run-test-' |
| 798 | regex += '(' + '|'.join(VARIANT_TYPE_DICT['run']) + ')-' |
| 799 | regex += '(' + '|'.join(VARIANT_TYPE_DICT['prebuild']) + ')-' |
| 800 | regex += '(' + '|'.join(VARIANT_TYPE_DICT['compiler']) + ')-' |
| 801 | regex += '(' + '|'.join(VARIANT_TYPE_DICT['relocate']) + ')-' |
| 802 | regex += '(' + '|'.join(VARIANT_TYPE_DICT['trace']) + ')-' |
| 803 | regex += '(' + '|'.join(VARIANT_TYPE_DICT['gc']) + ')-' |
| 804 | regex += '(' + '|'.join(VARIANT_TYPE_DICT['jni']) + ')-' |
| 805 | regex += '(' + '|'.join(VARIANT_TYPE_DICT['image']) + ')-' |
| 806 | regex += '(' + '|'.join(VARIANT_TYPE_DICT['pictest']) + ')-' |
| 807 | regex += '(' + '|'.join(VARIANT_TYPE_DICT['debuggable']) + ')-' |
Alex Light | 8f2c6d4 | 2017-04-10 16:27:35 -0700 | [diff] [blame] | 808 | regex += '(' + '|'.join(VARIANT_TYPE_DICT['jvmti']) + ')-' |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 809 | regex += '(' + '|'.join(RUN_TEST_SET) + ')' |
| 810 | regex += '(' + '|'.join(VARIANT_TYPE_DICT['address_sizes']) + ')$' |
| 811 | match = re.match(regex, test_name) |
| 812 | if match: |
| 813 | TARGET_TYPES.add(match.group(1)) |
| 814 | RUN_TYPES.add(match.group(2)) |
| 815 | PREBUILD_TYPES.add(match.group(3)) |
| 816 | COMPILER_TYPES.add(match.group(4)) |
| 817 | RELOCATE_TYPES.add(match.group(5)) |
| 818 | TRACE_TYPES.add(match.group(6)) |
| 819 | GC_TYPES.add(match.group(7)) |
| 820 | JNI_TYPES.add(match.group(8)) |
| 821 | IMAGE_TYPES.add(match.group(9)) |
| 822 | PICTEST_TYPES.add(match.group(10)) |
| 823 | DEBUGGABLE_TYPES.add(match.group(11)) |
Alex Light | 8f2c6d4 | 2017-04-10 16:27:35 -0700 | [diff] [blame] | 824 | JVMTI_TYPES.add(match.group(12)) |
| 825 | ADDRESS_SIZES.add(match.group(14)) |
| 826 | return {match.group(13)} |
Shubham Ajmera | faf1250 | 2017-02-15 17:19:44 +0000 | [diff] [blame] | 827 | raise ValueError(test_name + " is not a valid test") |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 828 | |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 829 | |
| 830 | def setup_env_for_build_target(build_target, parser, options): |
| 831 | """Setup environment for the build target |
| 832 | |
| 833 | The method setup environment for the master-art-host targets. |
| 834 | """ |
| 835 | os.environ.update(build_target['env']) |
| 836 | os.environ['SOONG_ALLOW_MISSING_DEPENDENCIES'] = 'true' |
| 837 | print_text('%s\n' % (str(os.environ))) |
| 838 | |
| 839 | target_options = vars(parser.parse_args(build_target['flags'])) |
| 840 | target_options['host'] = True |
| 841 | target_options['verbose'] = True |
| 842 | target_options['build'] = True |
| 843 | target_options['n_thread'] = options['n_thread'] |
| 844 | target_options['dry_run'] = options['dry_run'] |
| 845 | |
| 846 | return target_options |
| 847 | |
Shubham Ajmera | 4a5a162 | 2017-03-22 10:07:19 -0700 | [diff] [blame] | 848 | def get_default_threads(target): |
| 849 | if target is 'target': |
| 850 | adb_command = 'adb shell cat /sys/devices/system/cpu/present' |
| 851 | cpu_info_proc = subprocess.Popen(adb_command.split(), stdout=subprocess.PIPE) |
| 852 | cpu_info = cpu_info_proc.stdout.read() |
Shubham Ajmera | 8fd2694 | 2017-05-09 11:30:47 -0700 | [diff] [blame] | 853 | if type(cpu_info) is bytes: |
| 854 | cpu_info = cpu_info.decode('utf-8') |
| 855 | cpu_info_regex = '\d*-(\d*)' |
| 856 | match = re.match(cpu_info_regex, cpu_info) |
| 857 | if match: |
| 858 | return int(match.group(1)) |
| 859 | else: |
| 860 | raise ValueError('Unable to predict the concurrency for the target. ' |
| 861 | 'Is device connected?') |
Shubham Ajmera | 4a5a162 | 2017-03-22 10:07:19 -0700 | [diff] [blame] | 862 | else: |
| 863 | return multiprocessing.cpu_count() |
| 864 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 865 | def parse_option(): |
| 866 | global verbose |
| 867 | global dry_run |
| 868 | global n_thread |
| 869 | global build |
| 870 | global gdb |
| 871 | global gdb_arg |
Shubham Ajmera | fe79349 | 2017-03-16 13:31:35 -0700 | [diff] [blame] | 872 | global timeout |
Shubham Ajmera | 981d99c | 2017-08-17 14:11:08 -0700 | [diff] [blame] | 873 | global dex2oat_jobs |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 874 | |
Alex Light | 7a1ccf8 | 2017-02-21 09:52:34 -0800 | [diff] [blame] | 875 | parser = argparse.ArgumentParser(description="Runs all or a subset of the ART test suite.") |
| 876 | parser.add_argument('-t', '--test', dest='test', help='name of the test') |
| 877 | parser.add_argument('-j', type=int, dest='n_thread') |
Shubham Ajmera | 186d321 | 2017-07-21 01:20:57 +0000 | [diff] [blame] | 878 | parser.add_argument('--timeout', default=timeout, type=int, dest='timeout') |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 879 | for variant in TOTAL_VARIANTS_SET: |
| 880 | flag = '--' + variant |
| 881 | flag_dest = variant.replace('-', '_') |
| 882 | if variant == '32' or variant == '64': |
| 883 | flag_dest = 'n' + flag_dest |
Alex Light | 7a1ccf8 | 2017-02-21 09:52:34 -0800 | [diff] [blame] | 884 | parser.add_argument(flag, action='store_true', dest=flag_dest) |
| 885 | parser.add_argument('--verbose', '-v', action='store_true', dest='verbose') |
| 886 | parser.add_argument('--dry-run', action='store_true', dest='dry_run') |
| 887 | parser.add_argument("--skip", action="append", dest="skips", default=[], |
| 888 | help="Skip the given test in all circumstances.") |
Alex Light | 9b6b13e | 2017-02-22 11:46:50 -0800 | [diff] [blame] | 889 | parser.add_argument('--no-build-dependencies', |
| 890 | action='store_false', dest='build', |
| 891 | help="Don't build dependencies under any circumstances. This is the " + |
| 892 | "behavior if ART_TEST_RUN_TEST_ALWAYS_BUILD is not set to 'true'.") |
| 893 | parser.add_argument('-b', '--build-dependencies', |
| 894 | action='store_true', dest='build', |
| 895 | help="Build dependencies under all circumstances. By default we will " + |
| 896 | "not build dependencies unless ART_TEST_RUN_TEST_BUILD=true.") |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 897 | parser.add_argument('--build-target', dest='build_target', help='master-art-host targets') |
Alex Light | 9b6b13e | 2017-02-22 11:46:50 -0800 | [diff] [blame] | 898 | parser.set_defaults(build = env.ART_TEST_RUN_TEST_BUILD) |
Alex Light | 7a1ccf8 | 2017-02-21 09:52:34 -0800 | [diff] [blame] | 899 | parser.add_argument('--gdb', action='store_true', dest='gdb') |
| 900 | parser.add_argument('--gdb-arg', dest='gdb_arg') |
Shubham Ajmera | 981d99c | 2017-08-17 14:11:08 -0700 | [diff] [blame] | 901 | parser.add_argument('--dex2oat-jobs', type=int, dest='dex2oat_jobs', |
| 902 | help='Number of dex2oat jobs') |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 903 | |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 904 | options = vars(parser.parse_args()) |
| 905 | if options['build_target']: |
| 906 | options = setup_env_for_build_target(target_config[options['build_target']], |
| 907 | parser, options) |
| 908 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 909 | test = '' |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 910 | env.EXTRA_DISABLED_TESTS.update(set(options['skips'])) |
| 911 | if options['test']: |
| 912 | test = parse_test_name(options['test']) |
| 913 | if options['pictest']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 914 | PICTEST_TYPES.add('pictest') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 915 | if options['ndebug']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 916 | RUN_TYPES.add('ndebug') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 917 | if options['interp_ac']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 918 | COMPILER_TYPES.add('interp-ac') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 919 | if options['picimage']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 920 | IMAGE_TYPES.add('picimage') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 921 | if options['n64']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 922 | ADDRESS_SIZES.add('64') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 923 | if options['interpreter']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 924 | COMPILER_TYPES.add('interpreter') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 925 | if options['jni']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 926 | JNI_TYPES.add('jni') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 927 | if options['relocate_npatchoat']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 928 | RELOCATE_TYPES.add('relocate-npatchoat') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 929 | if options['no_prebuild']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 930 | PREBUILD_TYPES.add('no-prebuild') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 931 | if options['npictest']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 932 | PICTEST_TYPES.add('npictest') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 933 | if options['no_dex2oat']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 934 | PREBUILD_TYPES.add('no-dex2oat') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 935 | if options['jit']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 936 | COMPILER_TYPES.add('jit') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 937 | if options['relocate']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 938 | RELOCATE_TYPES.add('relocate') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 939 | if options['ndebuggable']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 940 | DEBUGGABLE_TYPES.add('ndebuggable') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 941 | if options['no_image']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 942 | IMAGE_TYPES.add('no-image') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 943 | if options['optimizing']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 944 | COMPILER_TYPES.add('optimizing') |
Jeff Hao | 002b931 | 2017-03-27 16:23:08 -0700 | [diff] [blame] | 945 | if options['speed_profile']: |
| 946 | COMPILER_TYPES.add('speed-profile') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 947 | if options['trace']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 948 | TRACE_TYPES.add('trace') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 949 | if options['gcstress']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 950 | GC_TYPES.add('gcstress') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 951 | if options['no_relocate']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 952 | RELOCATE_TYPES.add('no-relocate') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 953 | if options['target']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 954 | TARGET_TYPES.add('target') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 955 | if options['forcecopy']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 956 | JNI_TYPES.add('forcecopy') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 957 | if options['n32']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 958 | ADDRESS_SIZES.add('32') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 959 | if options['host']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 960 | TARGET_TYPES.add('host') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 961 | if options['gcverify']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 962 | GC_TYPES.add('gcverify') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 963 | if options['debuggable']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 964 | DEBUGGABLE_TYPES.add('debuggable') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 965 | if options['prebuild']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 966 | PREBUILD_TYPES.add('prebuild') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 967 | if options['debug']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 968 | RUN_TYPES.add('debug') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 969 | if options['checkjni']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 970 | JNI_TYPES.add('checkjni') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 971 | if options['ntrace']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 972 | TRACE_TYPES.add('ntrace') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 973 | if options['cms']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 974 | GC_TYPES.add('cms') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 975 | if options['multipicimage']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 976 | IMAGE_TYPES.add('multipicimage') |
Alex Light | 8f2c6d4 | 2017-04-10 16:27:35 -0700 | [diff] [blame] | 977 | if options['jvmti_stress']: |
| 978 | JVMTI_TYPES.add('jvmti-stress') |
Alex Light | b7edcda | 2017-04-27 13:20:31 -0700 | [diff] [blame] | 979 | if options['redefine_stress']: |
| 980 | JVMTI_TYPES.add('redefine-stress') |
Alex Light | 43e935d | 2017-06-19 15:40:40 -0700 | [diff] [blame] | 981 | if options['field_stress']: |
| 982 | JVMTI_TYPES.add('field-stress') |
Alex Light | c38c369 | 2017-06-27 15:45:14 -0700 | [diff] [blame] | 983 | if options['step_stress']: |
| 984 | JVMTI_TYPES.add('step-stress') |
Alex Light | b7edcda | 2017-04-27 13:20:31 -0700 | [diff] [blame] | 985 | if options['trace_stress']: |
| 986 | JVMTI_TYPES.add('trace-stress') |
Alex Light | 8f2c6d4 | 2017-04-10 16:27:35 -0700 | [diff] [blame] | 987 | if options['no_jvmti']: |
| 988 | JVMTI_TYPES.add('no-jvmti') |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 989 | if options['verbose']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 990 | verbose = True |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 991 | if options['n_thread']: |
| 992 | n_thread = max(1, options['n_thread']) |
| 993 | if options['dry_run']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 994 | dry_run = True |
| 995 | verbose = True |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 996 | build = options['build'] |
| 997 | if options['gdb']: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 998 | n_thread = 1 |
| 999 | gdb = True |
Shubham Ajmera | b9d09ca | 2017-03-07 10:45:05 -0800 | [diff] [blame] | 1000 | if options['gdb_arg']: |
| 1001 | gdb_arg = options['gdb_arg'] |
Shubham Ajmera | fe79349 | 2017-03-16 13:31:35 -0700 | [diff] [blame] | 1002 | timeout = options['timeout'] |
Shubham Ajmera | 981d99c | 2017-08-17 14:11:08 -0700 | [diff] [blame] | 1003 | if options['dex2oat_jobs']: |
| 1004 | dex2oat_jobs = options['dex2oat_jobs'] |
Shubham Ajmera | 22499e2 | 2017-03-22 18:33:37 -0700 | [diff] [blame] | 1005 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 1006 | return test |
| 1007 | |
| 1008 | def main(): |
| 1009 | gather_test_info() |
| 1010 | user_requested_test = parse_option() |
| 1011 | setup_test_env() |
| 1012 | if build: |
| 1013 | build_targets = '' |
| 1014 | if 'host' in TARGET_TYPES: |
Shubham Ajmera | 186d321 | 2017-07-21 01:20:57 +0000 | [diff] [blame] | 1015 | build_targets += 'test-art-host-run-test-dependencies' |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 1016 | if 'target' in TARGET_TYPES: |
Shubham Ajmera | 186d321 | 2017-07-21 01:20:57 +0000 | [diff] [blame] | 1017 | build_targets += 'test-art-target-run-test-dependencies' |
Shubham Ajmera | 06cde29 | 2017-02-10 23:15:05 +0000 | [diff] [blame] | 1018 | build_command = 'make' |
Shubham Ajmera | 4a5a162 | 2017-03-22 10:07:19 -0700 | [diff] [blame] | 1019 | build_command += ' -j' |
Shubham Ajmera | 06cde29 | 2017-02-10 23:15:05 +0000 | [diff] [blame] | 1020 | build_command += ' -C ' + env.ANDROID_BUILD_TOP |
| 1021 | build_command += ' ' + build_targets |
Nicolas Geoffray | 300c09b | 2017-03-22 12:26:32 +0000 | [diff] [blame] | 1022 | # Add 'dist' to avoid Jack issues b/36169180. |
| 1023 | build_command += ' dist' |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 1024 | if subprocess.call(build_command.split()): |
| 1025 | sys.exit(1) |
| 1026 | if user_requested_test: |
| 1027 | test_runner_thread = threading.Thread(target=run_tests, args=(user_requested_test,)) |
| 1028 | else: |
| 1029 | test_runner_thread = threading.Thread(target=run_tests, args=(RUN_TEST_SET,)) |
| 1030 | test_runner_thread.daemon = True |
| 1031 | try: |
| 1032 | test_runner_thread.start() |
| 1033 | while threading.active_count() > 1: |
Shubham Ajmera | 186d321 | 2017-07-21 01:20:57 +0000 | [diff] [blame] | 1034 | time.sleep(0.1) |
| 1035 | print_analysis() |
Shubham Ajmera | fe79349 | 2017-03-16 13:31:35 -0700 | [diff] [blame] | 1036 | except Exception as e: |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 1037 | print_analysis() |
Shubham Ajmera | faf1250 | 2017-02-15 17:19:44 +0000 | [diff] [blame] | 1038 | print_text(str(e)) |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 1039 | sys.exit(1) |
Shubham Ajmera | c5aae87 | 2017-02-16 19:58:59 +0000 | [diff] [blame] | 1040 | if failed_tests: |
| 1041 | sys.exit(1) |
| 1042 | sys.exit(0) |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 1043 | |
Shubham Ajmera | 65adb8b | 2017-02-06 16:04:25 +0000 | [diff] [blame] | 1044 | if __name__ == '__main__': |
| 1045 | main() |