The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2007 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package android.test; |
| 18 | |
Paul Duffin | fedb4b7 | 2017-06-22 10:47:25 +0100 | [diff] [blame] | 19 | import android.test.suitebuilder.annotation.MediumTest; |
| 20 | import android.test.suitebuilder.annotation.SmallTest; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 21 | import com.android.internal.util.Predicate; |
| 22 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 23 | import android.app.Activity; |
| 24 | import android.app.Instrumentation; |
| 25 | import android.os.Bundle; |
| 26 | import android.os.Debug; |
| 27 | import android.os.Looper; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 28 | import android.os.Parcelable; |
| 29 | import android.os.PerformanceCollector; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 30 | import android.os.PerformanceCollector.PerformanceResultsWriter; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 31 | import android.test.suitebuilder.TestMethod; |
| 32 | import android.test.suitebuilder.TestPredicates; |
| 33 | import android.test.suitebuilder.TestSuiteBuilder; |
Doug Zongker | 0375fa1 | 2010-02-23 12:36:06 -0800 | [diff] [blame] | 34 | import android.test.suitebuilder.annotation.LargeTest; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 35 | import android.util.Log; |
| 36 | |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 37 | import java.io.ByteArrayOutputStream; |
| 38 | import java.io.File; |
| 39 | import java.io.PrintStream; |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 40 | import java.lang.annotation.Annotation; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 41 | import java.lang.reflect.InvocationTargetException; |
| 42 | import java.lang.reflect.Method; |
| 43 | import java.util.ArrayList; |
| 44 | import java.util.List; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 45 | |
| 46 | import junit.framework.AssertionFailedError; |
| 47 | import junit.framework.Test; |
| 48 | import junit.framework.TestCase; |
| 49 | import junit.framework.TestListener; |
| 50 | import junit.framework.TestResult; |
| 51 | import junit.framework.TestSuite; |
| 52 | import junit.runner.BaseTestRunner; |
| 53 | import junit.textui.ResultPrinter; |
| 54 | |
Paul Duffin | fedb4b7 | 2017-06-22 10:47:25 +0100 | [diff] [blame] | 55 | import static android.test.suitebuilder.TestPredicates.hasAnnotation; |
| 56 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 57 | /** |
| 58 | * An {@link Instrumentation} that runs various types of {@link junit.framework.TestCase}s against |
Joe Fernandez | 3aef8e1d | 2011-12-20 10:38:34 -0800 | [diff] [blame] | 59 | * an Android package (application). |
| 60 | * |
| 61 | * <div class="special reference"> |
| 62 | * <h3>Developer Guides</h3> |
| 63 | * <p>For more information about application testing, read the |
| 64 | * <a href="{@docRoot}guide/topics/testing/index.html">Testing</a> developer guide.</p> |
| 65 | * </div> |
| 66 | * |
| 67 | * <h3>Typical Usage</h3> |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 68 | * <ol> |
| 69 | * <li>Write {@link junit.framework.TestCase}s that perform unit, functional, or performance tests |
| 70 | * against the classes in your package. Typically these are subclassed from: |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 71 | * <ul><li>{@link android.test.ActivityInstrumentationTestCase2}</li> |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 72 | * <li>{@link android.test.ActivityUnitTestCase}</li> |
| 73 | * <li>{@link android.test.AndroidTestCase}</li> |
| 74 | * <li>{@link android.test.ApplicationTestCase}</li> |
| 75 | * <li>{@link android.test.InstrumentationTestCase}</li> |
| 76 | * <li>{@link android.test.ProviderTestCase}</li> |
| 77 | * <li>{@link android.test.ServiceTestCase}</li> |
| 78 | * <li>{@link android.test.SingleLaunchActivityTestCase}</li></ul> |
quddusc | e8df6f4 | 2014-01-06 16:28:03 -0800 | [diff] [blame] | 79 | * <li>Set the <code>android:targetPackage</code> attribute of the <code><instrumentation></code> |
| 80 | * element in the test package's manifest. You should set the attribute value |
| 81 | * to the package name of the target application under test. |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 82 | * <li>Run the instrumentation using "adb shell am instrument -w", |
| 83 | * with no optional arguments, to run all tests (except performance tests). |
| 84 | * <li>Run the instrumentation using "adb shell am instrument -w", |
| 85 | * with the argument '-e func true' to run all functional tests. These are tests that derive from |
| 86 | * {@link android.test.InstrumentationTestCase}. |
| 87 | * <li>Run the instrumentation using "adb shell am instrument -w", |
| 88 | * with the argument '-e unit true' to run all unit tests. These are tests that <i>do not</i>derive |
| 89 | * from {@link android.test.InstrumentationTestCase} (and are not performance tests). |
| 90 | * <li>Run the instrumentation using "adb shell am instrument -w", |
| 91 | * with the argument '-e class' set to run an individual {@link junit.framework.TestCase}. |
| 92 | * </ol> |
| 93 | * <p/> |
| 94 | * <b>Running all tests:</b> adb shell am instrument -w |
| 95 | * com.android.foo/android.test.InstrumentationTestRunner |
| 96 | * <p/> |
| 97 | * <b>Running all small tests:</b> adb shell am instrument -w |
| 98 | * -e size small |
| 99 | * com.android.foo/android.test.InstrumentationTestRunner |
| 100 | * <p/> |
| 101 | * <b>Running all medium tests:</b> adb shell am instrument -w |
| 102 | * -e size medium |
| 103 | * com.android.foo/android.test.InstrumentationTestRunner |
| 104 | * <p/> |
| 105 | * <b>Running all large tests:</b> adb shell am instrument -w |
| 106 | * -e size large |
| 107 | * com.android.foo/android.test.InstrumentationTestRunner |
| 108 | * <p/> |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 109 | * <b>Filter test run to tests with given annotation:</b> adb shell am instrument -w |
| 110 | * -e annotation com.android.foo.MyAnnotation |
| 111 | * com.android.foo/android.test.InstrumentationTestRunner |
| 112 | * <p/> |
| 113 | * If used with other options, the resulting test run will contain the union of the two options. |
| 114 | * e.g. "-e size large -e annotation com.android.foo.MyAnnotation" will run only tests with both |
| 115 | * the {@link LargeTest} and "com.android.foo.MyAnnotation" annotations. |
| 116 | * <p/> |
| 117 | * <b>Filter test run to tests <i>without</i> given annotation:</b> adb shell am instrument -w |
| 118 | * -e notAnnotation com.android.foo.MyAnnotation |
| 119 | * com.android.foo/android.test.InstrumentationTestRunner |
| 120 | * <p/> |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 121 | * <b>Running a single testcase:</b> adb shell am instrument -w |
| 122 | * -e class com.android.foo.FooTest |
| 123 | * com.android.foo/android.test.InstrumentationTestRunner |
| 124 | * <p/> |
| 125 | * <b>Running a single test:</b> adb shell am instrument -w |
| 126 | * -e class com.android.foo.FooTest#testFoo |
| 127 | * com.android.foo/android.test.InstrumentationTestRunner |
| 128 | * <p/> |
| 129 | * <b>Running multiple tests:</b> adb shell am instrument -w |
| 130 | * -e class com.android.foo.FooTest,com.android.foo.TooTest |
| 131 | * com.android.foo/android.test.InstrumentationTestRunner |
| 132 | * <p/> |
Brett Chabot | 89c0ef4 | 2010-03-18 20:03:31 -0700 | [diff] [blame] | 133 | * <b>Running all tests in a java package:</b> adb shell am instrument -w |
| 134 | * -e package com.android.foo.subpkg |
| 135 | * com.android.foo/android.test.InstrumentationTestRunner |
| 136 | * <p/> |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 137 | * <b>Including performance tests:</b> adb shell am instrument -w |
| 138 | * -e perf true |
| 139 | * com.android.foo/android.test.InstrumentationTestRunner |
| 140 | * <p/> |
| 141 | * <b>To debug your tests, set a break point in your code and pass:</b> |
| 142 | * -e debug true |
| 143 | * <p/> |
| 144 | * <b>To run in 'log only' mode</b> |
| 145 | * -e log true |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 146 | * This option will load and iterate through all test classes and methods, but will bypass actual |
| 147 | * test execution. Useful for quickly obtaining info on the tests to be executed by an |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 148 | * instrumentation command. |
| 149 | * <p/> |
| 150 | * <b>To generate EMMA code coverage:</b> |
| 151 | * -e coverage true |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 152 | * Note: this requires an emma instrumented build. By default, the code coverage results file |
Brett Chabot | 51e0364 | 2009-05-28 18:18:15 -0700 | [diff] [blame] | 153 | * will be saved in a /data/<app>/coverage.ec file, unless overridden by coverageFile flag (see |
| 154 | * below) |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 155 | * <p/> |
| 156 | * <b> To specify EMMA code coverage results file path:</b> |
| 157 | * -e coverageFile /sdcard/myFile.ec |
| 158 | * <br/> |
| 159 | * in addition to the other arguments. |
Stephan Linzner | b51617f | 2016-01-27 18:09:50 -0800 | [diff] [blame] | 160 | * @deprecated Use |
| 161 | * <a href="{@docRoot}reference/android/support/test/runner/AndroidJUnitRunner.html"> |
| 162 | * AndroidJUnitRunner</a> instead. New tests should be written using the |
| 163 | * <a href="{@docRoot}tools/testing-support-library/index.html">Android Testing Support Library</a>. |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 164 | */ |
| 165 | |
| 166 | /* (not JavaDoc) |
| 167 | * Although not necessary in most case, another way to use this class is to extend it and have the |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 168 | * derived class return the desired test suite from the {@link #getTestSuite()} method. The test |
| 169 | * suite returned from this method will be used if no target class is defined in the meta-data or |
| 170 | * command line argument parameters. If a derived class is used it needs to be added as an |
| 171 | * instrumentation to the AndroidManifest.xml and the command to run it would look like: |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 172 | * <p/> |
| 173 | * adb shell am instrument -w com.android.foo/<i>com.android.FooInstrumentationTestRunner</i> |
| 174 | * <p/> |
| 175 | * Where <i>com.android.FooInstrumentationTestRunner</i> is the derived class. |
| 176 | * |
| 177 | * This model is used by many existing app tests, but can probably be deprecated. |
| 178 | */ |
Stephan Linzner | b51617f | 2016-01-27 18:09:50 -0800 | [diff] [blame] | 179 | @Deprecated |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 180 | public class InstrumentationTestRunner extends Instrumentation implements TestSuiteProvider { |
| 181 | |
| 182 | /** @hide */ |
Paul Duffin | 575f9da | 2017-06-20 14:41:20 +0100 | [diff] [blame] | 183 | static final String ARGUMENT_TEST_CLASS = "class"; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 184 | /** @hide */ |
Paul Duffin | 575f9da | 2017-06-20 14:41:20 +0100 | [diff] [blame] | 185 | private static final String ARGUMENT_TEST_PACKAGE = "package"; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 186 | /** @hide */ |
Paul Duffin | 575f9da | 2017-06-20 14:41:20 +0100 | [diff] [blame] | 187 | private static final String ARGUMENT_TEST_SIZE_PREDICATE = "size"; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 188 | /** @hide */ |
Paul Duffin | 575f9da | 2017-06-20 14:41:20 +0100 | [diff] [blame] | 189 | static final String ARGUMENT_DELAY_MSEC = "delay_msec"; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 190 | |
| 191 | private static final String SMALL_SUITE = "small"; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 192 | private static final String MEDIUM_SUITE = "medium"; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 193 | private static final String LARGE_SUITE = "large"; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 194 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 195 | private static final String ARGUMENT_LOG_ONLY = "log"; |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 196 | /** @hide */ |
| 197 | static final String ARGUMENT_ANNOTATION = "annotation"; |
| 198 | /** @hide */ |
| 199 | static final String ARGUMENT_NOT_ANNOTATION = "notAnnotation"; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 200 | |
Paul Duffin | fedb4b7 | 2017-06-22 10:47:25 +0100 | [diff] [blame] | 201 | private static final Predicate<TestMethod> SELECT_SMALL = hasAnnotation(SmallTest.class); |
| 202 | |
| 203 | private static final Predicate<TestMethod> SELECT_MEDIUM = hasAnnotation(MediumTest.class); |
| 204 | |
| 205 | private static final Predicate<TestMethod> SELECT_LARGE = hasAnnotation(LargeTest.class); |
| 206 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 207 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 208 | * This constant defines the maximum allowed runtime (in ms) for a test included in the "small" |
| 209 | * suite. It is used to make an educated guess at what suite an unlabeled test belongs. |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 210 | */ |
| 211 | private static final float SMALL_SUITE_MAX_RUNTIME = 100; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 212 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 213 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 214 | * This constant defines the maximum allowed runtime (in ms) for a test included in the |
| 215 | * "medium" suite. It is used to make an educated guess at what suite an unlabeled test belongs. |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 216 | */ |
| 217 | private static final float MEDIUM_SUITE_MAX_RUNTIME = 1000; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 218 | |
Paul Duffin | 575f9da | 2017-06-20 14:41:20 +0100 | [diff] [blame] | 219 | /* |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 220 | * The following keys are used in the status bundle to provide structured reports to |
| 221 | * an IInstrumentationWatcher. |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 222 | */ |
| 223 | |
| 224 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 225 | * This value, if stored with key {@link android.app.Instrumentation#REPORT_KEY_IDENTIFIER}, |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 226 | * identifies InstrumentationTestRunner as the source of the report. This is sent with all |
| 227 | * status messages. |
| 228 | */ |
| 229 | public static final String REPORT_VALUE_ID = "InstrumentationTestRunner"; |
| 230 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 231 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 232 | * identifies the total number of tests that are being run. This is sent with all status |
| 233 | * messages. |
| 234 | */ |
| 235 | public static final String REPORT_KEY_NUM_TOTAL = "numtests"; |
| 236 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 237 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 238 | * identifies the sequence number of the current test. This is sent with any status message |
| 239 | * describing a specific test being started or completed. |
| 240 | */ |
| 241 | public static final String REPORT_KEY_NUM_CURRENT = "current"; |
| 242 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 243 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 244 | * identifies the name of the current test class. This is sent with any status message |
| 245 | * describing a specific test being started or completed. |
| 246 | */ |
| 247 | public static final String REPORT_KEY_NAME_CLASS = "class"; |
| 248 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 249 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 250 | * identifies the name of the current test. This is sent with any status message |
| 251 | * describing a specific test being started or completed. |
| 252 | */ |
| 253 | public static final String REPORT_KEY_NAME_TEST = "test"; |
| 254 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 255 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 256 | * reports the run time in seconds of the current test. |
| 257 | */ |
| 258 | private static final String REPORT_KEY_RUN_TIME = "runtime"; |
| 259 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 260 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
Jack Wang | 3fc03e6 | 2010-10-19 15:13:07 -0700 | [diff] [blame] | 261 | * reports the number of total iterations of the current test. |
| 262 | */ |
| 263 | private static final String REPORT_KEY_NUM_ITERATIONS = "numiterations"; |
| 264 | /** |
| 265 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 266 | * reports the guessed suite assignment for the current test. |
| 267 | */ |
| 268 | private static final String REPORT_KEY_SUITE_ASSIGNMENT = "suiteassignment"; |
| 269 | /** |
Brett Chabot | 51e0364 | 2009-05-28 18:18:15 -0700 | [diff] [blame] | 270 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
| 271 | * identifies the path to the generated code coverage file. |
| 272 | */ |
| 273 | private static final String REPORT_KEY_COVERAGE_PATH = "coverageFilePath"; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 274 | |
| 275 | /** |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 276 | * The test is starting. |
| 277 | */ |
| 278 | public static final int REPORT_VALUE_RESULT_START = 1; |
| 279 | /** |
| 280 | * The test completed successfully. |
| 281 | */ |
| 282 | public static final int REPORT_VALUE_RESULT_OK = 0; |
| 283 | /** |
| 284 | * The test completed with an error. |
| 285 | */ |
| 286 | public static final int REPORT_VALUE_RESULT_ERROR = -1; |
| 287 | /** |
| 288 | * The test completed with a failure. |
| 289 | */ |
| 290 | public static final int REPORT_VALUE_RESULT_FAILURE = -2; |
| 291 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 292 | * If included in the status bundle sent to an IInstrumentationWatcher, this key |
| 293 | * identifies a stack trace describing an error or failure. This is sent with any status |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 294 | * message describing a specific test being completed. |
| 295 | */ |
| 296 | public static final String REPORT_KEY_STACK = "stack"; |
| 297 | |
Brett Chabot | 51e0364 | 2009-05-28 18:18:15 -0700 | [diff] [blame] | 298 | // Default file name for code coverage |
| 299 | private static final String DEFAULT_COVERAGE_FILE_NAME = "coverage.ec"; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 300 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 301 | private static final String LOG_TAG = "InstrumentationTestRunner"; |
| 302 | |
| 303 | private final Bundle mResults = new Bundle(); |
David Hu | cbc584d | 2011-09-16 18:08:35 -0700 | [diff] [blame] | 304 | private Bundle mArguments; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 305 | private AndroidTestRunner mTestRunner; |
| 306 | private boolean mDebug; |
| 307 | private boolean mJustCount; |
| 308 | private boolean mSuiteAssignmentMode; |
| 309 | private int mTestCount; |
| 310 | private String mPackageOfTests; |
| 311 | private boolean mCoverage; |
| 312 | private String mCoverageFilePath; |
| 313 | private int mDelayMsec; |
| 314 | |
| 315 | @Override |
| 316 | public void onCreate(Bundle arguments) { |
| 317 | super.onCreate(arguments); |
David Hu | cbc584d | 2011-09-16 18:08:35 -0700 | [diff] [blame] | 318 | mArguments = arguments; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 319 | |
| 320 | // Apk paths used to search for test classes when using TestSuiteBuilders. |
| 321 | String[] apkPaths = |
| 322 | {getTargetContext().getPackageCodePath(), getContext().getPackageCodePath()}; |
| 323 | ClassPathPackageInfoSource.setApkPaths(apkPaths); |
| 324 | |
| 325 | Predicate<TestMethod> testSizePredicate = null; |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 326 | Predicate<TestMethod> testAnnotationPredicate = null; |
| 327 | Predicate<TestMethod> testNotAnnotationPredicate = null; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 328 | String testClassesArg = null; |
| 329 | boolean logOnly = false; |
| 330 | |
| 331 | if (arguments != null) { |
| 332 | // Test class name passed as an argument should override any meta-data declaration. |
| 333 | testClassesArg = arguments.getString(ARGUMENT_TEST_CLASS); |
| 334 | mDebug = getBooleanArgument(arguments, "debug"); |
| 335 | mJustCount = getBooleanArgument(arguments, "count"); |
| 336 | mSuiteAssignmentMode = getBooleanArgument(arguments, "suiteAssignment"); |
| 337 | mPackageOfTests = arguments.getString(ARGUMENT_TEST_PACKAGE); |
| 338 | testSizePredicate = getSizePredicateFromArg( |
| 339 | arguments.getString(ARGUMENT_TEST_SIZE_PREDICATE)); |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 340 | testAnnotationPredicate = getAnnotationPredicate( |
| 341 | arguments.getString(ARGUMENT_ANNOTATION)); |
| 342 | testNotAnnotationPredicate = getNotAnnotationPredicate( |
| 343 | arguments.getString(ARGUMENT_NOT_ANNOTATION)); |
| 344 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 345 | logOnly = getBooleanArgument(arguments, ARGUMENT_LOG_ONLY); |
| 346 | mCoverage = getBooleanArgument(arguments, "coverage"); |
| 347 | mCoverageFilePath = arguments.getString("coverageFile"); |
| 348 | |
| 349 | try { |
| 350 | Object delay = arguments.get(ARGUMENT_DELAY_MSEC); // Accept either string or int |
| 351 | if (delay != null) mDelayMsec = Integer.parseInt(delay.toString()); |
| 352 | } catch (NumberFormatException e) { |
| 353 | Log.e(LOG_TAG, "Invalid delay_msec parameter", e); |
| 354 | } |
| 355 | } |
| 356 | |
| 357 | TestSuiteBuilder testSuiteBuilder = new TestSuiteBuilder(getClass().getName(), |
| 358 | getTargetContext().getClassLoader()); |
| 359 | |
| 360 | if (testSizePredicate != null) { |
| 361 | testSuiteBuilder.addRequirements(testSizePredicate); |
| 362 | } |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 363 | if (testAnnotationPredicate != null) { |
| 364 | testSuiteBuilder.addRequirements(testAnnotationPredicate); |
| 365 | } |
| 366 | if (testNotAnnotationPredicate != null) { |
| 367 | testSuiteBuilder.addRequirements(testNotAnnotationPredicate); |
| 368 | } |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 369 | |
| 370 | if (testClassesArg == null) { |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 371 | if (mPackageOfTests != null) { |
| 372 | testSuiteBuilder.includePackages(mPackageOfTests); |
| 373 | } else { |
Brett Chabot | 61b10ac | 2009-03-31 17:04:34 -0700 | [diff] [blame] | 374 | TestSuite testSuite = getTestSuite(); |
| 375 | if (testSuite != null) { |
| 376 | testSuiteBuilder.addTestSuite(testSuite); |
| 377 | } else { |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 378 | // no package or class bundle arguments were supplied, and no test suite |
Brett Chabot | 61b10ac | 2009-03-31 17:04:34 -0700 | [diff] [blame] | 379 | // provided so add all tests in application |
| 380 | testSuiteBuilder.includePackages(""); |
| 381 | } |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 382 | } |
| 383 | } else { |
| 384 | parseTestClasses(testClassesArg, testSuiteBuilder); |
| 385 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 386 | |
Urs Grob | da13ef5 | 2009-04-17 11:30:14 -0700 | [diff] [blame] | 387 | testSuiteBuilder.addRequirements(getBuilderRequirements()); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 388 | |
| 389 | mTestRunner = getAndroidTestRunner(); |
| 390 | mTestRunner.setContext(getTargetContext()); |
Jack Wang | 7aba54b | 2009-08-20 19:20:54 -0700 | [diff] [blame] | 391 | mTestRunner.setInstrumentation(this); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 392 | mTestRunner.setSkipExecution(logOnly); |
| 393 | mTestRunner.setTest(testSuiteBuilder.build()); |
| 394 | mTestCount = mTestRunner.getTestCases().size(); |
| 395 | if (mSuiteAssignmentMode) { |
| 396 | mTestRunner.addTestListener(new SuiteAssignmentPrinter()); |
| 397 | } else { |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 398 | WatcherResultPrinter resultPrinter = new WatcherResultPrinter(mTestCount); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 399 | mTestRunner.addTestListener(new TestPrinter("TestRunner", false)); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 400 | mTestRunner.addTestListener(resultPrinter); |
| 401 | mTestRunner.setPerformanceResultsWriter(resultPrinter); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 402 | } |
| 403 | start(); |
| 404 | } |
| 405 | |
David Hu | cbc584d | 2011-09-16 18:08:35 -0700 | [diff] [blame] | 406 | /** |
Svetoslav Ganov | 80943d8 | 2013-01-02 10:25:37 -0800 | [diff] [blame] | 407 | * Get the arguments passed to this instrumentation. |
David Hu | cbc584d | 2011-09-16 18:08:35 -0700 | [diff] [blame] | 408 | * |
| 409 | * @return the Bundle object |
David Hu | cbc584d | 2011-09-16 18:08:35 -0700 | [diff] [blame] | 410 | */ |
Svetoslav | 3a34d17 | 2013-01-28 19:55:35 -0800 | [diff] [blame] | 411 | public Bundle getArguments() { |
David Hu | cbc584d | 2011-09-16 18:08:35 -0700 | [diff] [blame] | 412 | return mArguments; |
| 413 | } |
| 414 | |
David Hu | 8cc9a8e | 2011-12-13 15:57:42 -0800 | [diff] [blame] | 415 | /** |
| 416 | * Add a {@link TestListener} |
| 417 | * @hide |
| 418 | */ |
| 419 | protected void addTestListener(TestListener listener){ |
| 420 | if(mTestRunner!=null && listener!=null){ |
| 421 | mTestRunner.addTestListener(listener); |
| 422 | } |
| 423 | } |
| 424 | |
Urs Grob | da13ef5 | 2009-04-17 11:30:14 -0700 | [diff] [blame] | 425 | List<Predicate<TestMethod>> getBuilderRequirements() { |
| 426 | return new ArrayList<Predicate<TestMethod>>(); |
| 427 | } |
| 428 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 429 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 430 | * Parses and loads the specified set of test classes |
| 431 | * |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 432 | * @param testClassArg - comma-separated list of test classes and methods |
| 433 | * @param testSuiteBuilder - builder to add tests to |
| 434 | */ |
| 435 | private void parseTestClasses(String testClassArg, TestSuiteBuilder testSuiteBuilder) { |
| 436 | String[] testClasses = testClassArg.split(","); |
| 437 | for (String testClass : testClasses) { |
| 438 | parseTestClass(testClass, testSuiteBuilder); |
| 439 | } |
| 440 | } |
| 441 | |
| 442 | /** |
| 443 | * Parse and load the given test class and, optionally, method |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 444 | * |
| 445 | * @param testClassName - full package name of test class and optionally method to add. |
| 446 | * Expected format: com.android.TestClass#testMethod |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 447 | * @param testSuiteBuilder - builder to add tests to |
| 448 | */ |
| 449 | private void parseTestClass(String testClassName, TestSuiteBuilder testSuiteBuilder) { |
| 450 | int methodSeparatorIndex = testClassName.indexOf('#'); |
| 451 | String testMethodName = null; |
| 452 | |
| 453 | if (methodSeparatorIndex > 0) { |
| 454 | testMethodName = testClassName.substring(methodSeparatorIndex + 1); |
| 455 | testClassName = testClassName.substring(0, methodSeparatorIndex); |
| 456 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 457 | testSuiteBuilder.addTestClassByName(testClassName, testMethodName, getTargetContext()); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 458 | } |
| 459 | |
| 460 | protected AndroidTestRunner getAndroidTestRunner() { |
| 461 | return new AndroidTestRunner(); |
| 462 | } |
| 463 | |
| 464 | private boolean getBooleanArgument(Bundle arguments, String tag) { |
| 465 | String tagString = arguments.getString(tag); |
| 466 | return tagString != null && Boolean.parseBoolean(tagString); |
| 467 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 468 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 469 | /* |
| 470 | * Returns the size predicate object, corresponding to the "size" argument value. |
| 471 | */ |
| 472 | private Predicate<TestMethod> getSizePredicateFromArg(String sizeArg) { |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 473 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 474 | if (SMALL_SUITE.equals(sizeArg)) { |
Paul Duffin | fedb4b7 | 2017-06-22 10:47:25 +0100 | [diff] [blame] | 475 | return SELECT_SMALL; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 476 | } else if (MEDIUM_SUITE.equals(sizeArg)) { |
Paul Duffin | fedb4b7 | 2017-06-22 10:47:25 +0100 | [diff] [blame] | 477 | return SELECT_MEDIUM; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 478 | } else if (LARGE_SUITE.equals(sizeArg)) { |
Paul Duffin | fedb4b7 | 2017-06-22 10:47:25 +0100 | [diff] [blame] | 479 | return SELECT_LARGE; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 480 | } else { |
| 481 | return null; |
| 482 | } |
| 483 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 484 | |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 485 | /** |
| 486 | * Returns the test predicate object, corresponding to the annotation class value provided via |
Paul Duffin | 575f9da | 2017-06-20 14:41:20 +0100 | [diff] [blame] | 487 | * the {@link #ARGUMENT_ANNOTATION} argument. |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 488 | * |
| 489 | * @return the predicate or <code>null</code> |
| 490 | */ |
| 491 | private Predicate<TestMethod> getAnnotationPredicate(String annotationClassName) { |
| 492 | Class<? extends Annotation> annotationClass = getAnnotationClass(annotationClassName); |
| 493 | if (annotationClass != null) { |
Paul Duffin | fedb4b7 | 2017-06-22 10:47:25 +0100 | [diff] [blame] | 494 | return hasAnnotation(annotationClass); |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 495 | } |
| 496 | return null; |
| 497 | } |
| 498 | |
| 499 | /** |
| 500 | * Returns the negative test predicate object, corresponding to the annotation class value |
Paul Duffin | 575f9da | 2017-06-20 14:41:20 +0100 | [diff] [blame] | 501 | * provided via the {@link #ARGUMENT_NOT_ANNOTATION} argument. |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 502 | * |
| 503 | * @return the predicate or <code>null</code> |
| 504 | */ |
| 505 | private Predicate<TestMethod> getNotAnnotationPredicate(String annotationClassName) { |
| 506 | Class<? extends Annotation> annotationClass = getAnnotationClass(annotationClassName); |
| 507 | if (annotationClass != null) { |
Paul Duffin | fedb4b7 | 2017-06-22 10:47:25 +0100 | [diff] [blame] | 508 | return TestPredicates.not(hasAnnotation(annotationClass)); |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 509 | } |
| 510 | return null; |
| 511 | } |
| 512 | |
| 513 | /** |
| 514 | * Helper method to return the annotation class with specified name |
| 515 | * |
| 516 | * @param annotationClassName the fully qualified name of the class |
| 517 | * @return the annotation class or <code>null</code> |
| 518 | */ |
| 519 | private Class<? extends Annotation> getAnnotationClass(String annotationClassName) { |
| 520 | if (annotationClassName == null) { |
| 521 | return null; |
| 522 | } |
| 523 | try { |
| 524 | Class<?> annotationClass = Class.forName(annotationClassName); |
| 525 | if (annotationClass.isAnnotation()) { |
| 526 | return (Class<? extends Annotation>)annotationClass; |
| 527 | } else { |
| 528 | Log.e(LOG_TAG, String.format("Provided annotation value %s is not an Annotation", |
| 529 | annotationClassName)); |
| 530 | } |
| 531 | } catch (ClassNotFoundException e) { |
| 532 | Log.e(LOG_TAG, String.format("Could not find class for specified annotation %s", |
| 533 | annotationClassName)); |
| 534 | } |
| 535 | return null; |
| 536 | } |
| 537 | |
Brett Chabot | 31e7ce7 | 2010-07-07 17:19:08 -0700 | [diff] [blame] | 538 | /** |
| 539 | * Initialize the current thread as a looper. |
| 540 | * <p/> |
| 541 | * Exposed for unit testing. |
| 542 | */ |
| 543 | void prepareLooper() { |
| 544 | Looper.prepare(); |
| 545 | } |
| 546 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 547 | @Override |
| 548 | public void onStart() { |
Brett Chabot | 31e7ce7 | 2010-07-07 17:19:08 -0700 | [diff] [blame] | 549 | prepareLooper(); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 550 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 551 | if (mJustCount) { |
| 552 | mResults.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID); |
| 553 | mResults.putInt(REPORT_KEY_NUM_TOTAL, mTestCount); |
| 554 | finish(Activity.RESULT_OK, mResults); |
| 555 | } else { |
| 556 | if (mDebug) { |
| 557 | Debug.waitForDebugger(); |
| 558 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 559 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 560 | ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); |
| 561 | PrintStream writer = new PrintStream(byteArrayOutputStream); |
| 562 | try { |
| 563 | StringResultPrinter resultPrinter = new StringResultPrinter(writer); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 564 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 565 | mTestRunner.addTestListener(resultPrinter); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 566 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 567 | long startTime = System.currentTimeMillis(); |
| 568 | mTestRunner.runTest(); |
| 569 | long runTime = System.currentTimeMillis() - startTime; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 570 | |
Brett Chabot | 41e173d | 2012-10-24 14:34:07 -0700 | [diff] [blame] | 571 | resultPrinter.printResult(mTestRunner.getTestResult(), runTime); |
Brett Chabot | 31e7ce7 | 2010-07-07 17:19:08 -0700 | [diff] [blame] | 572 | } catch (Throwable t) { |
| 573 | // catch all exceptions so a more verbose error message can be outputted |
| 574 | writer.println(String.format("Test run aborted due to unexpected exception: %s", |
| 575 | t.getMessage())); |
| 576 | t.printStackTrace(writer); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 577 | } finally { |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 578 | mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, |
| 579 | String.format("\nTest results for %s=%s", |
| 580 | mTestRunner.getTestClassName(), |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 581 | byteArrayOutputStream.toString())); |
| 582 | |
| 583 | if (mCoverage) { |
| 584 | generateCoverageReport(); |
| 585 | } |
| 586 | writer.close(); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 587 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 588 | finish(Activity.RESULT_OK, mResults); |
| 589 | } |
| 590 | } |
| 591 | } |
| 592 | |
| 593 | public TestSuite getTestSuite() { |
| 594 | return getAllTests(); |
| 595 | } |
| 596 | |
| 597 | /** |
| 598 | * Override this to define all of the tests to run in your package. |
| 599 | */ |
| 600 | public TestSuite getAllTests() { |
| 601 | return null; |
| 602 | } |
| 603 | |
| 604 | /** |
| 605 | * Override this to provide access to the class loader of your package. |
| 606 | */ |
| 607 | public ClassLoader getLoader() { |
| 608 | return null; |
| 609 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 610 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 611 | private void generateCoverageReport() { |
| 612 | // use reflection to call emma dump coverage method, to avoid |
| 613 | // always statically compiling against emma jar |
Brett Chabot | 51e0364 | 2009-05-28 18:18:15 -0700 | [diff] [blame] | 614 | String coverageFilePath = getCoverageFilePath(); |
| 615 | java.io.File coverageFile = new java.io.File(coverageFilePath); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 616 | try { |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 617 | Class<?> emmaRTClass = Class.forName("com.vladium.emma.rt.RT"); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 618 | Method dumpCoverageMethod = emmaRTClass.getMethod("dumpCoverageData", |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 619 | coverageFile.getClass(), boolean.class, boolean.class); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 620 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 621 | dumpCoverageMethod.invoke(null, coverageFile, false, false); |
Brett Chabot | 51e0364 | 2009-05-28 18:18:15 -0700 | [diff] [blame] | 622 | // output path to generated coverage file so it can be parsed by a test harness if |
| 623 | // needed |
| 624 | mResults.putString(REPORT_KEY_COVERAGE_PATH, coverageFilePath); |
| 625 | // also output a more user friendly msg |
Brett Chabot | 08d13c3 | 2010-02-18 15:42:13 -0800 | [diff] [blame] | 626 | final String currentStream = mResults.getString( |
| 627 | Instrumentation.REPORT_KEY_STREAMRESULT); |
Brett Chabot | 51e0364 | 2009-05-28 18:18:15 -0700 | [diff] [blame] | 628 | mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, |
Brett Chabot | 08d13c3 | 2010-02-18 15:42:13 -0800 | [diff] [blame] | 629 | String.format("%s\nGenerated code coverage data to %s", currentStream, |
| 630 | coverageFilePath)); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 631 | } catch (ClassNotFoundException e) { |
| 632 | reportEmmaError("Is emma jar on classpath?", e); |
| 633 | } catch (SecurityException e) { |
| 634 | reportEmmaError(e); |
| 635 | } catch (NoSuchMethodException e) { |
| 636 | reportEmmaError(e); |
| 637 | } catch (IllegalArgumentException e) { |
| 638 | reportEmmaError(e); |
| 639 | } catch (IllegalAccessException e) { |
| 640 | reportEmmaError(e); |
| 641 | } catch (InvocationTargetException e) { |
| 642 | reportEmmaError(e); |
| 643 | } |
| 644 | } |
| 645 | |
| 646 | private String getCoverageFilePath() { |
| 647 | if (mCoverageFilePath == null) { |
Brett Chabot | 51e0364 | 2009-05-28 18:18:15 -0700 | [diff] [blame] | 648 | return getTargetContext().getFilesDir().getAbsolutePath() + File.separator + |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 649 | DEFAULT_COVERAGE_FILE_NAME; |
| 650 | } else { |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 651 | return mCoverageFilePath; |
| 652 | } |
| 653 | } |
| 654 | |
| 655 | private void reportEmmaError(Exception e) { |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 656 | reportEmmaError("", e); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 657 | } |
| 658 | |
| 659 | private void reportEmmaError(String hint, Exception e) { |
| 660 | String msg = "Failed to generate emma coverage. " + hint; |
| 661 | Log.e(LOG_TAG, msg, e); |
| 662 | mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "\nError: " + msg); |
| 663 | } |
| 664 | |
| 665 | // TODO kill this, use status() and prettyprint model for better output |
| 666 | private class StringResultPrinter extends ResultPrinter { |
| 667 | |
| 668 | public StringResultPrinter(PrintStream writer) { |
| 669 | super(writer); |
| 670 | } |
| 671 | |
Brett Chabot | 41e173d | 2012-10-24 14:34:07 -0700 | [diff] [blame] | 672 | public synchronized void printResult(TestResult result, long runTime) { |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 673 | printHeader(runTime); |
| 674 | printFooter(result); |
| 675 | } |
| 676 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 677 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 678 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 679 | * This class sends status reports back to the IInstrumentationWatcher about |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 680 | * which suite each test belongs. |
| 681 | */ |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 682 | private class SuiteAssignmentPrinter implements TestListener { |
| 683 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 684 | private Bundle mTestResult; |
| 685 | private long mStartTime; |
| 686 | private long mEndTime; |
| 687 | private boolean mTimingValid; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 688 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 689 | public SuiteAssignmentPrinter() { |
| 690 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 691 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 692 | /** |
| 693 | * send a status for the start of a each test, so long tests can be seen as "running" |
| 694 | */ |
| 695 | public void startTest(Test test) { |
| 696 | mTimingValid = true; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 697 | mStartTime = System.currentTimeMillis(); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 698 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 699 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 700 | /** |
| 701 | * @see junit.framework.TestListener#addError(Test, Throwable) |
| 702 | */ |
| 703 | public void addError(Test test, Throwable t) { |
| 704 | mTimingValid = false; |
| 705 | } |
| 706 | |
| 707 | /** |
| 708 | * @see junit.framework.TestListener#addFailure(Test, AssertionFailedError) |
| 709 | */ |
| 710 | public void addFailure(Test test, AssertionFailedError t) { |
| 711 | mTimingValid = false; |
| 712 | } |
| 713 | |
| 714 | /** |
| 715 | * @see junit.framework.TestListener#endTest(Test) |
| 716 | */ |
| 717 | public void endTest(Test test) { |
| 718 | float runTime; |
| 719 | String assignmentSuite; |
| 720 | mEndTime = System.currentTimeMillis(); |
| 721 | mTestResult = new Bundle(); |
| 722 | |
| 723 | if (!mTimingValid || mStartTime < 0) { |
| 724 | assignmentSuite = "NA"; |
| 725 | runTime = -1; |
| 726 | } else { |
| 727 | runTime = mEndTime - mStartTime; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 728 | if (runTime < SMALL_SUITE_MAX_RUNTIME |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 729 | && !InstrumentationTestCase.class.isAssignableFrom(test.getClass())) { |
| 730 | assignmentSuite = SMALL_SUITE; |
| 731 | } else if (runTime < MEDIUM_SUITE_MAX_RUNTIME) { |
| 732 | assignmentSuite = MEDIUM_SUITE; |
| 733 | } else { |
| 734 | assignmentSuite = LARGE_SUITE; |
| 735 | } |
| 736 | } |
| 737 | // Clear mStartTime so that we can verify that it gets set next time. |
| 738 | mStartTime = -1; |
| 739 | |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 740 | mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, |
| 741 | test.getClass().getName() + "#" + ((TestCase) test).getName() |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 742 | + "\nin " + assignmentSuite + " suite\nrunTime: " |
| 743 | + String.valueOf(runTime) + "\n"); |
| 744 | mTestResult.putFloat(REPORT_KEY_RUN_TIME, runTime); |
| 745 | mTestResult.putString(REPORT_KEY_SUITE_ASSIGNMENT, assignmentSuite); |
| 746 | |
| 747 | sendStatus(0, mTestResult); |
| 748 | } |
| 749 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 750 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 751 | /** |
| 752 | * This class sends status reports back to the IInstrumentationWatcher |
| 753 | */ |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 754 | private class WatcherResultPrinter implements TestListener, PerformanceResultsWriter { |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 755 | private final Bundle mResultTemplate; |
| 756 | Bundle mTestResult; |
| 757 | int mTestNum = 0; |
| 758 | int mTestResultCode = 0; |
| 759 | String mTestClass = null; |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 760 | PerformanceCollector mPerfCollector = new PerformanceCollector(); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 761 | boolean mIsTimedTest = false; |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 762 | boolean mIncludeDetailedStats = false; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 763 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 764 | public WatcherResultPrinter(int numTests) { |
| 765 | mResultTemplate = new Bundle(); |
| 766 | mResultTemplate.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID); |
| 767 | mResultTemplate.putInt(REPORT_KEY_NUM_TOTAL, numTests); |
| 768 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 769 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 770 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 771 | * send a status for the start of a each test, so long tests can be seen |
| 772 | * as "running" |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 773 | */ |
| 774 | public void startTest(Test test) { |
| 775 | String testClass = test.getClass().getName(); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 776 | String testName = ((TestCase)test).getName(); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 777 | mTestResult = new Bundle(mResultTemplate); |
| 778 | mTestResult.putString(REPORT_KEY_NAME_CLASS, testClass); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 779 | mTestResult.putString(REPORT_KEY_NAME_TEST, testName); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 780 | mTestResult.putInt(REPORT_KEY_NUM_CURRENT, ++mTestNum); |
| 781 | // pretty printing |
| 782 | if (testClass != null && !testClass.equals(mTestClass)) { |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 783 | mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 784 | String.format("\n%s:", testClass)); |
| 785 | mTestClass = testClass; |
| 786 | } else { |
| 787 | mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, ""); |
| 788 | } |
| 789 | |
Jack Wang | 3fc03e6 | 2010-10-19 15:13:07 -0700 | [diff] [blame] | 790 | Method testMethod = null; |
| 791 | try { |
| 792 | testMethod = test.getClass().getMethod(testName); |
| 793 | // Report total number of iterations, if test is repetitive |
| 794 | if (testMethod.isAnnotationPresent(RepetitiveTest.class)) { |
| 795 | int numIterations = testMethod.getAnnotation( |
| 796 | RepetitiveTest.class).numIterations(); |
| 797 | mTestResult.putInt(REPORT_KEY_NUM_ITERATIONS, numIterations); |
| 798 | } |
| 799 | } catch (NoSuchMethodException e) { |
| 800 | // ignore- the test with given name does not exist. Will be handled during test |
| 801 | // execution |
| 802 | } |
| 803 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 804 | // The delay_msec parameter is normally used to provide buffers of idle time |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 805 | // for power measurement purposes. To make sure there is a delay before and after |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 806 | // every test in a suite, we delay *after* every test (see endTest below) and also |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 807 | // delay *before* the first test. So, delay test1 delay test2 delay. |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 808 | |
| 809 | try { |
| 810 | if (mTestNum == 1) Thread.sleep(mDelayMsec); |
| 811 | } catch (InterruptedException e) { |
| 812 | throw new IllegalStateException(e); |
| 813 | } |
| 814 | |
| 815 | sendStatus(REPORT_VALUE_RESULT_START, mTestResult); |
| 816 | mTestResultCode = 0; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 817 | |
| 818 | mIsTimedTest = false; |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 819 | mIncludeDetailedStats = false; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 820 | try { |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 821 | // Look for TimedTest annotation on both test class and test method |
Brian Muramatsu | 4898395 | 2010-11-17 12:28:51 -0800 | [diff] [blame] | 822 | if (testMethod != null && testMethod.isAnnotationPresent(TimedTest.class)) { |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 823 | mIsTimedTest = true; |
Jack Wang | 3fc03e6 | 2010-10-19 15:13:07 -0700 | [diff] [blame] | 824 | mIncludeDetailedStats = testMethod.getAnnotation( |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 825 | TimedTest.class).includeDetailedStats(); |
| 826 | } else if (test.getClass().isAnnotationPresent(TimedTest.class)) { |
| 827 | mIsTimedTest = true; |
| 828 | mIncludeDetailedStats = test.getClass().getAnnotation( |
| 829 | TimedTest.class).includeDetailedStats(); |
| 830 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 831 | } catch (SecurityException e) { |
Brett Chabot | 31e7ce7 | 2010-07-07 17:19:08 -0700 | [diff] [blame] | 832 | // ignore - the test with given name cannot be accessed. Will be handled during |
| 833 | // test execution |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 834 | } |
| 835 | |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 836 | if (mIsTimedTest && mIncludeDetailedStats) { |
| 837 | mPerfCollector.beginSnapshot(""); |
| 838 | } else if (mIsTimedTest) { |
| 839 | mPerfCollector.startTiming(""); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 840 | } |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 841 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 842 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 843 | /** |
| 844 | * @see junit.framework.TestListener#addError(Test, Throwable) |
| 845 | */ |
| 846 | public void addError(Test test, Throwable t) { |
| 847 | mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t)); |
| 848 | mTestResultCode = REPORT_VALUE_RESULT_ERROR; |
| 849 | // pretty printing |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 850 | mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, |
| 851 | String.format("\nError in %s:\n%s", |
| 852 | ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t))); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 853 | } |
| 854 | |
| 855 | /** |
| 856 | * @see junit.framework.TestListener#addFailure(Test, AssertionFailedError) |
| 857 | */ |
| 858 | public void addFailure(Test test, AssertionFailedError t) { |
| 859 | mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t)); |
| 860 | mTestResultCode = REPORT_VALUE_RESULT_FAILURE; |
| 861 | // pretty printing |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 862 | mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, |
| 863 | String.format("\nFailure in %s:\n%s", |
| 864 | ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t))); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 865 | } |
| 866 | |
| 867 | /** |
| 868 | * @see junit.framework.TestListener#endTest(Test) |
| 869 | */ |
| 870 | public void endTest(Test test) { |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 871 | if (mIsTimedTest && mIncludeDetailedStats) { |
| 872 | mTestResult.putAll(mPerfCollector.endSnapshot()); |
| 873 | } else if (mIsTimedTest) { |
| 874 | writeStopTiming(mPerfCollector.stopTiming("")); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 875 | } |
| 876 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 877 | if (mTestResultCode == 0) { |
| 878 | mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "."); |
| 879 | } |
| 880 | sendStatus(mTestResultCode, mTestResult); |
| 881 | |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 882 | try { // Sleep after every test, if specified |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 883 | Thread.sleep(mDelayMsec); |
| 884 | } catch (InterruptedException e) { |
| 885 | throw new IllegalStateException(e); |
| 886 | } |
| 887 | } |
| 888 | |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 889 | public void writeBeginSnapshot(String label) { |
| 890 | // Do nothing |
| 891 | } |
| 892 | |
| 893 | public void writeEndSnapshot(Bundle results) { |
Jack Wang | 075997f | 2009-10-27 22:01:09 -0700 | [diff] [blame] | 894 | // Copy all snapshot data fields into mResults, which is outputted |
| 895 | // via Instrumentation.finish |
| 896 | mResults.putAll(results); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 897 | } |
| 898 | |
| 899 | public void writeStartTiming(String label) { |
| 900 | // Do nothing |
| 901 | } |
| 902 | |
| 903 | public void writeStopTiming(Bundle results) { |
| 904 | // Copy results into mTestResult by flattening list of iterations, |
| 905 | // which is outputted via WatcherResultPrinter.endTest |
| 906 | int i = 0; |
| 907 | for (Parcelable p : |
| 908 | results.getParcelableArrayList(PerformanceCollector.METRIC_KEY_ITERATIONS)) { |
| 909 | Bundle iteration = (Bundle)p; |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 910 | String index = "iteration" + i + "."; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 911 | mTestResult.putString(index + PerformanceCollector.METRIC_KEY_LABEL, |
| 912 | iteration.getString(PerformanceCollector.METRIC_KEY_LABEL)); |
| 913 | mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_CPU_TIME, |
| 914 | iteration.getLong(PerformanceCollector.METRIC_KEY_CPU_TIME)); |
| 915 | mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_EXECUTION_TIME, |
| 916 | iteration.getLong(PerformanceCollector.METRIC_KEY_EXECUTION_TIME)); |
| 917 | i++; |
| 918 | } |
| 919 | } |
| 920 | |
Jack Wang | 075997f | 2009-10-27 22:01:09 -0700 | [diff] [blame] | 921 | public void writeMeasurement(String label, long value) { |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 922 | mTestResult.putLong(label, value); |
Jack Wang | 075997f | 2009-10-27 22:01:09 -0700 | [diff] [blame] | 923 | } |
| 924 | |
| 925 | public void writeMeasurement(String label, float value) { |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 926 | mTestResult.putFloat(label, value); |
Jack Wang | 075997f | 2009-10-27 22:01:09 -0700 | [diff] [blame] | 927 | } |
| 928 | |
| 929 | public void writeMeasurement(String label, String value) { |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 930 | mTestResult.putString(label, value); |
Jack Wang | 075997f | 2009-10-27 22:01:09 -0700 | [diff] [blame] | 931 | } |
| 932 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 933 | // TODO report the end of the cycle |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 934 | } |
| 935 | } |