ART Test Runner
The testrunner runs the ART run-tests by simply invoking the script.
It fetches the list of eligible tests from art/test directory, and list of
disabled tests from art/test/knownfailures.json. It runs the tests by
invoking art/test/run-test script and checks the exit code to decide if
the test passed or failed.
Before invoking the script, first build all the tests dependencies by
building 'test-art-host-run-test-dependencies' for host tests,
'test-art-target-run-test-dependencies' for target tests, and
'test-art-run-test-dependencies' for building dependencies for both.
There are various options to invoke the script which are:
-t: Either the test name as in art/test or the test name including the variant
information. Eg, "-t 001-HelloWorld",
"-t test-art-host-run-test-debug-prebuild-optimizing-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32"
-j: Number of thread workers to be used. Eg - "-j64"
--dry-run: Instead of running the test name, just print its name.
--verbose
-b To build the dependencies before running the test.
To specify any specific variants for the test, use --<<variant-name>>.
For eg, for compiler type as optimizing, use --optimizing.
In the end, the script will print the failed and skipped tests if any.
New targets - test-art-host-run-test-dependencies and
test-art-target-run-test-dependencies have been added. The targets build
dependencies for host and target respectively.
Test: No tests
Change-Id: Ia4f13ee3444c2c733342c046ef1582517459fc9b
diff --git a/test/knownfailures.json b/test/knownfailures.json
new file mode 100644
index 0000000..98d56d4
--- /dev/null
+++ b/test/knownfailures.json
@@ -0,0 +1,365 @@
+[
+ {
+ "test": "153-reference-stress",
+ "description": ["Disable 153-reference-stress temporarily until a fix",
+ "arrives."],
+ "bug": "http://b/33389022"
+ },
+ {
+ "test": "080-oom-fragmentation",
+ "description": "Disable 080-oom-fragmentation due to flakes.",
+ "bug": "http://b/33795328"
+ },
+ {
+ "tests": ["497-inlining-and-class-loader",
+ "542-unresolved-access-check"],
+ "description": ["Disable 497-inlining-and-class-loader and ",
+ "542-unresolved-access-check until they are rewritten.",
+ "These tests use a broken class loader that tries to",
+ "register a dex file that's already registered with a",
+ "different loader."],
+ "bug": "http://b/34193123"
+ },
+ {
+ "test": "149-suspend-all-stress",
+ "description": "Disable 149-suspend-all-stress, its output is flaky",
+ "bug": "http://b/28988206"
+ },
+ {
+ "test": "577-profile-foreign-dex",
+ "description": "Disable 577-profile-foreign-dex",
+ "bug": "http://b/27454772"
+ },
+ {
+ "tests": ["002-sleep",
+ "053-wait-some",
+ "055-enum-performance",
+ "133-static-invoke-super"],
+ "description": ["Tests that are timing sensitive and flaky on heavily",
+ "loaded systems."]
+ },
+ {
+ "test": "147-stripped-dex-fallback",
+ "variant": "target",
+ "description": ["147-stripped-dex-fallback isn't supported on device",
+ "because --strip-dex requires the zip command."]
+ },
+ {
+ "test": "569-checker-pattern-replacement",
+ "variant": "target",
+ "description": ["569-checker-pattern-replacement tests behaviour",
+ "present only on host."]
+ },
+ {
+ "tests": ["116-nodex2oat",
+ "118-noimage-dex2oat",
+ "134-nodex2oat-nofallback"],
+ "variant": "prebuild",
+ "description": ["Note 116-nodex2oat is not broken per-se it just",
+ "doesn't (and isn't meant to) work with --prebuild."]
+ },
+ {
+ "test": "554-jit-profile-file",
+ "variant": "no-prebuild | interpreter",
+ "description": ["554-jit-profile-file is disabled because it needs a",
+ "primary oat file to know what it should save."]
+ },
+ {
+ "tests": ["529-checker-unresolved", "555-checker-regression-x86const"],
+ "variant": "no-prebuild",
+ "bug": "http://b/27784033"
+ },
+ {
+ "tests": ["117-nopatchoat",
+ "147-stripped-dex-fallback",
+ "608-checker-unresolved-lse"],
+ "variant": "no-prebuild"
+ },
+ {
+ "tests": ["117-nopatchoat",
+ "118-noimage-dex2oat",
+ "119-noimage-patchoat",
+ "554-jit-profile-file"],
+ "variant": "no-relocate",
+ "description": ["117-nopatchoat is not broken per-se it just doesn't",
+ "work (and isn't meant to) without --prebuild",
+ "--relocate"]
+ },
+ {
+ "test": "137-cfi",
+ "variant": "interp-ac",
+ "description": ["Temporarily disable some broken tests when forcing",
+ "access checks in interpreter"],
+ "bug": "http://b/22414682"
+ },
+ {
+ "test" : "629-vdex-speed",
+ "variant": "interp-ac | no-dex2oat | interpreter | jit | relocate-npatchoat",
+ "description": "629 requires compilation."
+ },
+ {
+ "test": "137-cfi",
+ "variant": "gcstress",
+ "description": ["137-cfi needs to unwind a second forked process. We're",
+ "using a primitive sleep to wait till we hope the",
+ "second process got into the expected state. The",
+ "slowness of gcstress makes this bad."]
+ },
+ {
+ "tests": ["908-gc-start-finish",
+ "913-heaps"],
+ "variant": "gcstress",
+ "description": ["908-gc-start-finish expects GCs only to be run at",
+ "clear points. The reduced heap size makes this",
+ "non-deterministic. Same for 913."]
+ },
+ {
+ "test": "961-default-iface-resolution-gen",
+ "variant": "gcstress",
+ "description": ["961-default-iface-resolution-gen and",
+ "964-default-iface-init-genare very long tests that",
+ "often will take more than the timeout to run when",
+ "gcstress is enabled. This is because gcstress slows",
+ "down allocations significantly which these tests do a",
+ "lot."]
+ },
+ {
+ "tests": ["964-default-iface-init-gen",
+ "154-gc-loop"],
+ "variant": "gcstress"
+ },
+ {
+ "test": "115-native-bridge",
+ "variant": "target",
+ "description": ["115-native-bridge setup is complicated. Need to",
+ "implement it correctly for the target."]
+ },
+ {
+ "test": "130-hprof",
+ "variant": "target",
+ "desription": ["130-hprof dumps the heap and runs hprof-conv to check",
+ "whether the file is somewhat readable. Thi is only",
+ "possible on the host. TODO: Turn off all the other",
+ "combinations, this is more about testing actual ART",
+ "code. A gtest is very hard to write here, as (for a",
+ "complete test) JDWP must be set up."]
+ },
+ {
+ "test": "131-structural-change",
+ "variant": "debug",
+ "description": ["131 is an old test. The functionality has been",
+ "implemented at an earlier stage and is checked",
+ "in tests 138. Blacklisted for debug builds since",
+ "these builds have duplicate classes checks which",
+ "punt to interpreter"]
+ },
+ {
+ "test": "138-duplicate-classes-check",
+ "variant": "ndebug",
+ "description": ["Turned on for debug builds since debug builds have",
+ "duplicate classes checks enabled"],
+ "bug": "http://b/2133391"
+ },
+ {
+ "test": "147-stripped-dex-fallback",
+ "variant": "no-dex2oat | no-image | relocate-npatchoat",
+ "description": ["147-stripped-dex-fallback is disabled because it",
+ "requires --prebuild."]
+ },
+ {
+ "test": "554-jit-profile-file",
+ "variant": "no-dex2oat | no-image | relocate-npatchoat",
+ "description": ["554-jit-profile-file is disabled because it needs a",
+ "primary oat file to know what it should save."]
+ },
+ {
+ "tests": ["116-nodex2oat",
+ "117-nopatchoat",
+ "118-noimage-dex2oat",
+ "119-noimage-patchoat",
+ "137-cfi",
+ "138-duplicate-classes-check2"],
+ "variant": "no-dex2oat | no-image | relocate-npatchoat",
+ "description": ["All these tests check that we have sane behavior if we",
+ "don't have a patchoat or dex2oat. Therefore we",
+ "shouldn't run them in situations where we actually",
+ "don't have these since they explicitly test for them.",
+ "These all also assume we have an image."]
+ },
+ {
+ "tests": ["137-cfi",
+ "138-duplicate-classes-check",
+ "018-stack-overflow",
+ "961-default-iface-resolution-gen",
+ "964-default-iface-init"],
+ "variant": "no-image",
+ "description": ["This test fails without an image. 018, 961, 964 often",
+ "time out."],
+ "bug": "http://b/34369284"
+ },
+ {
+ "test": "137-cfi",
+ "description": ["This test unrolls and expects managed frames, but",
+ "tracing means we run the interpreter."],
+ "variant": "trace | stream"
+ },
+ {
+ "tests": ["802-deoptimization",
+ "570-checker-osr"],
+ "description": ["This test dynamically enables tracing to force a",
+ "deoptimization. This makes the test meaningless",
+ "when already tracing, and writes an error message",
+ "that we do not want to check for."],
+ "variant": "trace | stream"
+ },
+ {
+ "test": "130-hprof",
+ "description": "130 occasional timeout",
+ "bug": "http://b/32383962",
+ "variant": "trace | stream"
+ },
+ {
+ "tests": ["087-gc-after-link",
+ "141-class-unload"],
+ "variant": "trace | stream"
+ },
+ {
+ "tests": ["604-hot-static-interface",
+ "612-jit-dex-cache",
+ "613-inlining-dex-cache",
+ "616-cha",
+ "626-set-resolved-string"],
+ "variant": "trace | stream",
+ "description": ["These tests expect JIT compilation, which is",
+ "suppressed when tracing."]
+ },
+ {
+ "test": "137-cfi",
+ "description": ["CFI unwinding expects managed frames, and the test",
+ "does not iterate enough to even compile. JIT also",
+ "uses Generic JNI instead of the JNI compiler."],
+ "variant": "interpreter | jit"
+ },
+ {
+ "test": "902-hello-transformation",
+ "description": "Test 902 hits races with the JIT compiler.",
+ "bug": "http://b/32821077",
+ "variant": "jit"
+ },
+ {
+ "test": "906-iterate-heap",
+ "description": ["Test 906 iterates the heap filtering with different",
+ "options. No instances should be created between those",
+ "runs to be able to have precise checks."],
+ "variant": "jit"
+ },
+ {
+ "tests": ["914-hello-obsolescence",
+ "915-obsolete-2",
+ "917-fields-transformation",
+ "919-obsolete-fields"],
+ "description": ["Test 914, 915, 917, & 919 are very sensitive to the",
+ "exact state of the stack, including the jit-inserted",
+ "runtime frames. This causes them to be somewhat flaky",
+ "as JIT tests. This should be fixed once b/33630159 or",
+ "b/33616143 are resolved but until then just disable",
+ "them. Test 916 already checks this feature for JIT",
+ "use cases in a way that is resilient to the jit",
+ "frames."],
+ "variant": "jit"
+ },
+ {
+ "tests": ["926-multi-obsolescence",
+ "904-object-allocation"],
+ "variant": "jit"
+ },
+ {
+ "test": "912-classes",
+ "variant": "jit",
+ "bug": "http://b/34655682"
+ },
+ {
+ "tests": ["570-checker-select",
+ "484-checker-register-hints"],
+ "description": ["These tests were based on the linear scan allocator,",
+ "which makes different decisions than the graph",
+ "coloring allocator. (These attempt to test for code",
+ "quality, not correctness.)"],
+ "variant": "regalloc_gc"
+ },
+ {
+ "tests": ["454-get-vreg",
+ "457-regs",
+ "602-deoptimizeable"],
+ "description": ["Tests that should fail when the optimizing compiler ",
+ "compiles them non-debuggable."],
+ "variant": "optimizing & ndebuggable | regalloc_gc & ndebuggable"
+ },
+ {
+ "test": "596-app-images",
+ "variant": "npictest"
+ },
+ {
+ "test": "055-enum-performance",
+ "variant": "optimizing | regalloc_gc",
+ "description": ["055: Exceeds run time limits due to heap poisoning ",
+ "instrumentation (on ARM and ARM64 devices)."]
+ },
+ {
+ "test": "909-attach-agent",
+ "variant": "debuggable",
+ "description": "Tests that check semantics for a non-debuggable app."
+ },
+ {
+ "test": "137-cfi",
+ "variant": "debuggable",
+ "description": ["The test relies on AOT code and debuggable makes us",
+ "JIT always."]
+ },
+ {
+ "tests": ["000-nop",
+ "134-nodex2oat-nofallback",
+ "147-stripped-dex-fallback",
+ "595-profile-saving"],
+ "description": "The doesn't compile anything",
+ "env_vars": {"ART_TEST_BISECTION": "true"},
+ "variant": "optimizing | regalloc_gc"
+ },
+ {
+ "tests": "089-many-methods",
+ "description": "The test tests a build failure",
+ "env_vars": {"ART_TEST_BISECTION": "true"},
+ "variant": "optimizing | regalloc_gc"
+ },
+ {
+ "tests": ["018-stack-overflow",
+ "116-nodex2oat",
+ "117-nopatchoat",
+ "118-noimage-dex2oat",
+ "119-noimage-patchoat",
+ "126-miranda-multidex",
+ "137-cfi"],
+ "description": "The test run dalvikvm more than once.",
+ "env_vars": {"ART_TEST_BISECTION": "true"},
+ "variant": "optimizing | regalloc_gc"
+ },
+ {
+ "tests": ["115-native-bridge",
+ "088-monitor-verification"],
+ "description": "The test assume they are always compiled.",
+ "env_vars": {"ART_TEST_BISECTION": "true"},
+ "variant": "optimizing | regalloc_gc"
+ },
+ {
+ "test": "055-enum-performance",
+ "description": ["The test tests performance which degrades during",
+ "bisecting."],
+ "env_vars": {"ART_TEST_BISECTION": "true"},
+ "variant": "optimizing | regalloc_gc"
+ },
+ {
+ "test": "537-checker-arraycopy",
+ "env_vars": {"ART_USE_READ_BARRIER": "true"},
+ "variant": "interpreter | optimizing | regalloc_gc | jit"
+ }
+]