Jim Tang | 815b889 | 2018-07-11 12:57:30 +0800 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # |
| 3 | # Copyright 2018, The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
| 17 | """ |
| 18 | Atest Argument Parser class for atest. |
| 19 | """ |
| 20 | |
| 21 | import argparse |
| 22 | |
kellyhung | bf73711 | 2019-01-18 17:01:36 +0800 | [diff] [blame] | 23 | import atest_utils |
Jim Tang | 815b889 | 2018-07-11 12:57:30 +0800 | [diff] [blame] | 24 | import constants |
| 25 | |
| 26 | |
| 27 | class AtestArgParser(argparse.ArgumentParser): |
| 28 | """Atest wrapper of ArgumentParser.""" |
| 29 | |
| 30 | def __init__(self): |
| 31 | """Initialise an ArgumentParser instance.""" |
kellyhung | bf73711 | 2019-01-18 17:01:36 +0800 | [diff] [blame] | 32 | atest_utils.print_data_collection_notice() |
Jim Tang | 815b889 | 2018-07-11 12:57:30 +0800 | [diff] [blame] | 33 | super(AtestArgParser, self).__init__( |
| 34 | description=constants.HELP_DESC, |
| 35 | epilog=self.EPILOG_TEXT, |
| 36 | formatter_class=argparse.RawTextHelpFormatter) |
| 37 | |
| 38 | def add_atest_args(self): |
| 39 | """A function that does ArgumentParser.add_argument()""" |
| 40 | self.add_argument('tests', nargs='*', help='Tests to build and/or run.') |
| 41 | self.add_argument('-b', '--build', action='append_const', dest='steps', |
| 42 | const=constants.BUILD_STEP, help='Run a build.') |
| 43 | self.add_argument('-i', '--install', action='append_const', dest='steps', |
| 44 | const=constants.INSTALL_STEP, help='Install an APK.') |
yangbill | cc1a21f | 2018-12-12 20:03:12 +0800 | [diff] [blame] | 45 | self.add_argument('--info', action='store_true', |
nelsonli | edbd745 | 2018-08-27 11:11:11 +0800 | [diff] [blame] | 46 | help='Show module information.') |
yangbill | cc1a21f | 2018-12-12 20:03:12 +0800 | [diff] [blame] | 47 | self.add_argument('--dry-run', action='store_true', |
| 48 | help='Dry run atest without building, installing and running ' |
| 49 | 'tests in real.') |
Jim Tang | 815b889 | 2018-07-11 12:57:30 +0800 | [diff] [blame] | 50 | self.add_argument('-t', '--test', action='append_const', dest='steps', |
| 51 | const=constants.TEST_STEP, |
| 52 | help='Run the tests. WARNING: Many test configs force cleanup ' |
| 53 | 'of device after test run. In this case, -d must be used in ' |
| 54 | 'previous test run to disable cleanup, for -t to work. ' |
| 55 | 'Otherwise, device will need to be setup again with -i.') |
| 56 | self.add_argument('-s', '--serial', help='The device to run the test on.') |
easoncylee | f0fb2b1 | 2019-01-22 15:49:09 +0800 | [diff] [blame] | 57 | self.add_argument('-L', '--list-modules', help='List testable modules for the given suite.') |
Jim Tang | 815b889 | 2018-07-11 12:57:30 +0800 | [diff] [blame] | 58 | self.add_argument('-d', '--disable-teardown', action='store_true', |
| 59 | help='Disables test teardown and cleanup.') |
| 60 | self.add_argument('-m', constants.REBUILD_MODULE_INFO_FLAG, action='store_true', |
| 61 | help='Forces a rebuild of the module-info.json file. ' |
| 62 | 'This may be necessary following a repo sync or ' |
| 63 | 'when writing a new test.') |
| 64 | self.add_argument('-w', '--wait-for-debugger', action='store_true', |
| 65 | help='Only for instrumentation tests. Waits for ' |
| 66 | 'debugger prior to execution.') |
| 67 | self.add_argument('-v', '--verbose', action='store_true', |
| 68 | help='Display DEBUG level logging.') |
| 69 | self.add_argument('-a', '--all-abi', action='store_true', |
| 70 | help='Set to run tests for all abi.') |
| 71 | self.add_argument('--generate-baseline', nargs='?', type=int, const=5, default=0, |
| 72 | help='Generate baseline metrics, run 5 iterations by default. ' |
| 73 | 'Provide an int argument to specify # iterations.') |
| 74 | self.add_argument('--generate-new-metrics', nargs='?', type=int, const=5, default=0, |
| 75 | help='Generate new metrics, run 5 iterations by default. ' |
| 76 | 'Provide an int argument to specify # iterations.') |
| 77 | self.add_argument('--detect-regression', nargs='*', |
| 78 | help='Run regression detection algorithm. Supply ' |
| 79 | 'path to baseline and/or new metrics folders.') |
Julien Desprez | d516865 | 2019-02-06 14:49:01 -0800 | [diff] [blame] | 80 | # Options related to module parameterization |
| 81 | self.add_argument('--instant', action='store_true', |
| 82 | help='Run the instant_app version of the module, ' |
| 83 | 'if the module supports it. Note: running a test ' |
| 84 | 'that does not support instant with --instant ' |
| 85 | 'will result in nothing running.') |
Jim Tang | 815b889 | 2018-07-11 12:57:30 +0800 | [diff] [blame] | 86 | # Options related to Test Mapping |
| 87 | self.add_argument('-p', '--test-mapping', action='store_true', |
| 88 | help='Run tests in TEST_MAPPING files.') |
| 89 | self.add_argument('--include-subdirs', action='store_true', |
| 90 | help='Include tests in TEST_MAPPING files in sub directories.') |
kellyhung | 0625d17 | 2018-06-21 16:40:27 +0800 | [diff] [blame] | 91 | # Options related to deviceless testing. |
| 92 | self.add_argument('--host', action='store_true', |
| 93 | help='Run the test completely on the host without ' |
| 94 | 'a device. (Note: running a host test that ' |
| 95 | 'requires a device with --host will fail.)') |
Jim Tang | 815b889 | 2018-07-11 12:57:30 +0800 | [diff] [blame] | 96 | # This arg actually doesn't consume anything, it's primarily used for the |
| 97 | # help description and creating custom_args in the NameSpace object. |
| 98 | self.add_argument('--', dest='custom_args', nargs='*', |
| 99 | help='Specify custom args for the test runners. ' |
| 100 | 'Everything after -- will be consumed as custom args.') |
| 101 | |
| 102 | def get_args(self): |
| 103 | """This method is to get args from actions and return optional args. |
| 104 | |
| 105 | Returns: |
| 106 | A list of optional arguments. |
| 107 | """ |
| 108 | argument_list = [] |
| 109 | # The output of _get_optional_actions(): [['-t', '--test'], [--info]] |
| 110 | # return an argument list: ['-t', '--test', '--info'] |
| 111 | for arg in self._get_optional_actions(): |
| 112 | argument_list.extend(arg.option_strings) |
| 113 | return argument_list |
| 114 | |
| 115 | |
| 116 | EPILOG_TEXT = ''' |
| 117 | |
| 118 | - - - - - - - - - |
| 119 | IDENTIFYING TESTS |
| 120 | - - - - - - - - - |
| 121 | |
| 122 | The positional argument <tests> should be a reference to one or more |
| 123 | of the tests you'd like to run. Multiple tests can be run in one command by |
| 124 | separating test references with spaces. |
| 125 | |
| 126 | Usage template: atest <reference_to_test_1> <reference_to_test_2> |
| 127 | |
| 128 | A <reference_to_test> can be satisfied by the test's MODULE NAME, |
| 129 | MODULE:CLASS, CLASS NAME, TF INTEGRATION TEST, FILE PATH or PACKAGE NAME. |
| 130 | Explanations and examples of each follow. |
| 131 | |
| 132 | |
| 133 | < MODULE NAME > |
| 134 | |
| 135 | Identifying a test by its module name will run the entire module. Input |
| 136 | the name as it appears in the LOCAL_MODULE or LOCAL_PACKAGE_NAME |
| 137 | variables in that test's Android.mk or Android.bp file. |
| 138 | |
| 139 | Note: Use < TF INTEGRATION TEST > to run non-module tests integrated |
| 140 | directly into TradeFed. |
| 141 | |
| 142 | Examples: |
| 143 | atest FrameworksServicesTests |
| 144 | atest CtsJankDeviceTestCases |
| 145 | |
| 146 | |
| 147 | < MODULE:CLASS > |
| 148 | |
| 149 | Identifying a test by its class name will run just the tests in that |
| 150 | class and not the whole module. MODULE:CLASS is the preferred way to run |
| 151 | a single class. MODULE is the same as described above. CLASS is the |
| 152 | name of the test class in the .java file. It can either be the fully |
| 153 | qualified class name or just the basic name. |
| 154 | |
| 155 | Examples: |
| 156 | atest FrameworksServicesTests:ScreenDecorWindowTests |
| 157 | atest FrameworksServicesTests:com.android.server.wm.ScreenDecorWindowTests |
| 158 | atest CtsJankDeviceTestCases:CtsDeviceJankUi |
| 159 | |
| 160 | |
| 161 | < CLASS NAME > |
| 162 | |
| 163 | A single class can also be run by referencing the class name without |
| 164 | the module name. |
| 165 | |
| 166 | Examples: |
| 167 | atest ScreenDecorWindowTests |
| 168 | atest CtsDeviceJankUi |
| 169 | |
| 170 | However, this will take more time than the equivalent MODULE:CLASS |
| 171 | reference, so we suggest using a MODULE:CLASS reference whenever |
| 172 | possible. Examples below are ordered by performance from the fastest |
| 173 | to the slowest: |
| 174 | |
| 175 | Examples: |
| 176 | atest FrameworksServicesTests:com.android.server.wm.ScreenDecorWindowTests |
| 177 | atest FrameworksServicesTests:ScreenDecorWindowTests |
| 178 | atest ScreenDecorWindowTests |
| 179 | |
| 180 | < TF INTEGRATION TEST > |
| 181 | |
| 182 | To run tests that are integrated directly into TradeFed (non-modules), |
| 183 | input the name as it appears in the output of the "tradefed.sh list |
| 184 | configs" cmd. |
| 185 | |
| 186 | Examples: |
| 187 | atest example/reboot |
| 188 | atest native-benchmark |
| 189 | |
| 190 | |
| 191 | < FILE PATH > |
| 192 | |
| 193 | Both module-based tests and integration-based tests can be run by |
| 194 | inputting the path to their test file or dir as appropriate. A single |
| 195 | class can also be run by inputting the path to the class's java file. |
| 196 | Both relative and absolute paths are supported. |
| 197 | |
| 198 | Example - 2 ways to run the `CtsJankDeviceTestCases` module via path: |
| 199 | 1. run module from android <repo root>: |
| 200 | atest cts/tests/jank/jank |
| 201 | |
| 202 | 2. from <android root>/cts/tests/jank: |
| 203 | atest . |
| 204 | |
| 205 | Example - run a specific class within CtsJankDeviceTestCases module |
| 206 | from <android repo> root via path: |
| 207 | atest cts/tests/jank/src/android/jank/cts/ui/CtsDeviceJankUi.java |
| 208 | |
| 209 | Example - run an integration test from <android repo> root via path: |
| 210 | atest tools/tradefederation/contrib/res/config/example/reboot.xml |
| 211 | |
| 212 | |
| 213 | < PACKAGE NAME > |
| 214 | |
| 215 | Atest supports searching tests from package name as well. |
| 216 | |
| 217 | Examples: |
| 218 | atest com.android.server.wm |
| 219 | atest android.jank.cts |
| 220 | |
| 221 | |
| 222 | - - - - - - - - - - - - - - - - - - - - - - - - - - |
| 223 | SPECIFYING INDIVIDUAL STEPS: BUILD, INSTALL OR RUN |
| 224 | - - - - - - - - - - - - - - - - - - - - - - - - - - |
| 225 | |
| 226 | The -b, -i and -t options allow you to specify which steps you want to run. |
| 227 | If none of those options are given, then all steps are run. If any of these |
| 228 | options are provided then only the listed steps are run. |
| 229 | |
| 230 | Note: -i alone is not currently support and can only be included with -t. |
| 231 | Both -b and -t can be run alone. |
| 232 | |
| 233 | Examples: |
| 234 | atest -b <test> (just build targets) |
| 235 | atest -t <test> (run tests only) |
| 236 | atest -it <test> (install apk and run tests) |
| 237 | atest -bt <test> (build targets, run tests, but skip installing apk) |
| 238 | |
| 239 | |
| 240 | Atest now has the ability to force a test to skip its cleanup/teardown step. |
| 241 | Many tests, e.g. CTS, cleanup the device after the test is run, so trying to |
| 242 | rerun your test with -t will fail without having the --disable-teardown |
| 243 | parameter. Use -d before -t to skip the test clean up step and test iteratively. |
| 244 | |
| 245 | atest -d <test> (disable installing apk and cleanning up device) |
| 246 | atest -t <test> |
| 247 | |
| 248 | Note that -t disables both setup/install and teardown/cleanup of the |
| 249 | device. So you can continue to rerun your test with just |
| 250 | |
| 251 | atest -t <test> |
| 252 | |
| 253 | as many times as you want. |
| 254 | |
| 255 | |
| 256 | - - - - - - - - - - - - - |
| 257 | RUNNING SPECIFIC METHODS |
| 258 | - - - - - - - - - - - - - |
| 259 | |
| 260 | It is possible to run only specific methods within a test class. To run |
| 261 | only specific methods, identify the class in any of the ways supported for |
| 262 | identifying a class (MODULE:CLASS, FILE PATH, etc) and then append the |
| 263 | name of the method or method using the following template: |
| 264 | |
| 265 | <reference_to_class>#<method1> |
| 266 | |
| 267 | Multiple methods can be specified with commas: |
| 268 | |
| 269 | <reference_to_class>#<method1>,<method2>,<method3>... |
| 270 | |
| 271 | Examples: |
| 272 | atest com.android.server.wm.ScreenDecorWindowTests#testMultipleDecors |
| 273 | |
| 274 | atest FrameworksServicesTests:ScreenDecorWindowTests#testFlagChange,testRemoval |
| 275 | |
| 276 | |
| 277 | - - - - - - - - - - - - - |
| 278 | RUNNING MULTIPLE CLASSES |
| 279 | - - - - - - - - - - - - - |
| 280 | |
| 281 | To run multiple classes, deliminate them with spaces just like you would |
| 282 | when running multiple tests. Atest will handle building and running |
| 283 | classes in the most efficient way possible, so specifying a subset of |
| 284 | classes in a module will improve performance over running the whole module. |
| 285 | |
| 286 | |
| 287 | Examples: |
| 288 | - two classes in same module: |
| 289 | atest FrameworksServicesTests:ScreenDecorWindowTests FrameworksServicesTests:DimmerTests |
| 290 | |
| 291 | - two classes, different modules: |
| 292 | atest FrameworksServicesTests:ScreenDecorWindowTests CtsJankDeviceTestCases:CtsDeviceJankUi |
| 293 | |
| 294 | |
| 295 | - - - - - - - - - - - |
| 296 | REGRESSION DETECTION |
| 297 | - - - - - - - - - - - |
| 298 | |
| 299 | Generate pre-patch or post-patch metrics without running regression detection: |
| 300 | |
| 301 | Example: |
| 302 | atest <test> --generate-baseline <optional iter> |
| 303 | atest <test> --generate-new-metrics <optional iter> |
| 304 | |
| 305 | Local regression detection can be run in three options: |
| 306 | |
| 307 | 1) Provide a folder containing baseline (pre-patch) metrics (generated |
| 308 | previously). Atest will run the tests n (default 5) iterations, generate |
| 309 | a new set of post-patch metrics, and compare those against existing metrics. |
| 310 | |
| 311 | Example: |
| 312 | atest <test> --detect-regression </path/to/baseline> --generate-new-metrics <optional iter> |
| 313 | |
| 314 | 2) Provide a folder containing post-patch metrics (generated previously). |
| 315 | Atest will run the tests n (default 5) iterations, generate a new set of |
| 316 | pre-patch metrics, and compare those against those provided. Note: the |
| 317 | developer needs to revert the device/tests to pre-patch state to generate |
| 318 | baseline metrics. |
| 319 | |
| 320 | Example: |
| 321 | atest <test> --detect-regression </path/to/new> --generate-baseline <optional iter> |
| 322 | |
| 323 | 3) Provide 2 folders containing both pre-patch and post-patch metrics. Atest |
| 324 | will run no tests but the regression detection algorithm. |
| 325 | |
| 326 | Example: |
| 327 | atest --detect-regression </path/to/baseline> </path/to/new> |
| 328 | |
| 329 | |
| 330 | - - - - - - - - - - - - |
| 331 | TESTS IN TEST MAPPING |
| 332 | - - - - - - - - - - - - |
| 333 | |
| 334 | Atest can run tests in TEST_MAPPING files: |
| 335 | |
| 336 | 1) Run presubmit tests in TEST_MAPPING files in current and parent |
| 337 | directories. You can also specify a target directory. |
| 338 | |
| 339 | Example: |
| 340 | atest (run presubmit tests in TEST_MAPPING files in current and parent directories) |
| 341 | atest --test-mapping </path/to/project> |
| 342 | (run presubmit tests in TEST_MAPPING files in </path/to/project> and its parent directories) |
| 343 | |
| 344 | 2) Run a specified test group in TEST_MAPPING files. |
| 345 | |
| 346 | Example: |
| 347 | atest :postsubmit |
| 348 | (run postsubmit tests in TEST_MAPPING files in current and parent directories) |
| 349 | atest :all |
| 350 | (Run tests from all groups in TEST_MAPPING files) |
| 351 | atest --test-mapping </path/to/project>:postsubmit |
| 352 | (run postsubmit tests in TEST_MAPPING files in </path/to/project> and its parent directories) |
| 353 | |
| 354 | 3) Run tests in TEST_MAPPING files including sub directories |
| 355 | |
| 356 | By default, atest will only search for tests in TEST_MAPPING files in |
| 357 | current (or given directory) and its parent directories. If you want to run |
| 358 | tests in TEST_MAPPING files in the sub-directories, you can use option |
| 359 | --include-subdirs to force atest to include those tests too. |
| 360 | |
| 361 | Example: |
| 362 | atest --include-subdirs [optional </path/to/project>:<test_group_name>] |
| 363 | (run presubmit tests in TEST_MAPPING files in current, sub and parent directories) |
| 364 | A path can be provided optionally if you want to search for tests in a give |
| 365 | directory, with optional test group name. By default, the test group is |
| 366 | presubmit. |
| 367 | |
nelsonli | e3f90de | 2018-06-22 14:59:39 +0800 | [diff] [blame] | 368 | ''' |