The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2007 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package android.test; |
| 18 | |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 19 | import com.android.internal.util.Predicate; |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 20 | import com.android.internal.util.Predicates; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 21 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 22 | import android.app.Activity; |
| 23 | import android.app.Instrumentation; |
| 24 | import android.os.Bundle; |
| 25 | import android.os.Debug; |
| 26 | import android.os.Looper; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 27 | import android.os.Parcelable; |
| 28 | import android.os.PerformanceCollector; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 29 | import android.os.PerformanceCollector.PerformanceResultsWriter; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 30 | import android.test.suitebuilder.TestMethod; |
| 31 | import android.test.suitebuilder.TestPredicates; |
| 32 | import android.test.suitebuilder.TestSuiteBuilder; |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 33 | import android.test.suitebuilder.annotation.HasAnnotation; |
Doug Zongker | 0375fa1 | 2010-02-23 12:36:06 -0800 | [diff] [blame] | 34 | import android.test.suitebuilder.annotation.LargeTest; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 35 | import android.util.Log; |
| 36 | |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 37 | import java.io.ByteArrayOutputStream; |
| 38 | import java.io.File; |
| 39 | import java.io.PrintStream; |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 40 | import java.lang.annotation.Annotation; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 41 | import java.lang.reflect.InvocationTargetException; |
| 42 | import java.lang.reflect.Method; |
| 43 | import java.util.ArrayList; |
| 44 | import java.util.List; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 45 | |
| 46 | import junit.framework.AssertionFailedError; |
| 47 | import junit.framework.Test; |
| 48 | import junit.framework.TestCase; |
| 49 | import junit.framework.TestListener; |
| 50 | import junit.framework.TestResult; |
| 51 | import junit.framework.TestSuite; |
| 52 | import junit.runner.BaseTestRunner; |
| 53 | import junit.textui.ResultPrinter; |
| 54 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 55 | /** |
| 56 | * An {@link Instrumentation} that runs various types of {@link junit.framework.TestCase}s against |
Joe Fernandez | 3aef8e1d | 2011-12-20 10:38:34 -0800 | [diff] [blame] | 57 | * an Android package (application). |
| 58 | * |
| 59 | * <div class="special reference"> |
| 60 | * <h3>Developer Guides</h3> |
| 61 | * <p>For more information about application testing, read the |
| 62 | * <a href="{@docRoot}guide/topics/testing/index.html">Testing</a> developer guide.</p> |
| 63 | * </div> |
| 64 | * |
| 65 | * <h3>Typical Usage</h3> |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 66 | * <ol> |
| 67 | * <li>Write {@link junit.framework.TestCase}s that perform unit, functional, or performance tests |
| 68 | * against the classes in your package. Typically these are subclassed from: |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 69 | * <ul><li>{@link android.test.ActivityInstrumentationTestCase2}</li> |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 70 | * <li>{@link android.test.ActivityUnitTestCase}</li> |
| 71 | * <li>{@link android.test.AndroidTestCase}</li> |
| 72 | * <li>{@link android.test.ApplicationTestCase}</li> |
| 73 | * <li>{@link android.test.InstrumentationTestCase}</li> |
| 74 | * <li>{@link android.test.ProviderTestCase}</li> |
| 75 | * <li>{@link android.test.ServiceTestCase}</li> |
| 76 | * <li>{@link android.test.SingleLaunchActivityTestCase}</li></ul> |
| 77 | * <li>In an appropriate AndroidManifest.xml, define the this instrumentation with |
| 78 | * the appropriate android:targetPackage set. |
| 79 | * <li>Run the instrumentation using "adb shell am instrument -w", |
| 80 | * with no optional arguments, to run all tests (except performance tests). |
| 81 | * <li>Run the instrumentation using "adb shell am instrument -w", |
| 82 | * with the argument '-e func true' to run all functional tests. These are tests that derive from |
| 83 | * {@link android.test.InstrumentationTestCase}. |
| 84 | * <li>Run the instrumentation using "adb shell am instrument -w", |
| 85 | * with the argument '-e unit true' to run all unit tests. These are tests that <i>do not</i>derive |
| 86 | * from {@link android.test.InstrumentationTestCase} (and are not performance tests). |
| 87 | * <li>Run the instrumentation using "adb shell am instrument -w", |
| 88 | * with the argument '-e class' set to run an individual {@link junit.framework.TestCase}. |
| 89 | * </ol> |
| 90 | * <p/> |
| 91 | * <b>Running all tests:</b> adb shell am instrument -w |
| 92 | * com.android.foo/android.test.InstrumentationTestRunner |
| 93 | * <p/> |
| 94 | * <b>Running all small tests:</b> adb shell am instrument -w |
| 95 | * -e size small |
| 96 | * com.android.foo/android.test.InstrumentationTestRunner |
| 97 | * <p/> |
| 98 | * <b>Running all medium tests:</b> adb shell am instrument -w |
| 99 | * -e size medium |
| 100 | * com.android.foo/android.test.InstrumentationTestRunner |
| 101 | * <p/> |
| 102 | * <b>Running all large tests:</b> adb shell am instrument -w |
| 103 | * -e size large |
| 104 | * com.android.foo/android.test.InstrumentationTestRunner |
| 105 | * <p/> |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 106 | * <b>Filter test run to tests with given annotation:</b> adb shell am instrument -w |
| 107 | * -e annotation com.android.foo.MyAnnotation |
| 108 | * com.android.foo/android.test.InstrumentationTestRunner |
| 109 | * <p/> |
| 110 | * If used with other options, the resulting test run will contain the union of the two options. |
| 111 | * e.g. "-e size large -e annotation com.android.foo.MyAnnotation" will run only tests with both |
| 112 | * the {@link LargeTest} and "com.android.foo.MyAnnotation" annotations. |
| 113 | * <p/> |
| 114 | * <b>Filter test run to tests <i>without</i> given annotation:</b> adb shell am instrument -w |
| 115 | * -e notAnnotation com.android.foo.MyAnnotation |
| 116 | * com.android.foo/android.test.InstrumentationTestRunner |
| 117 | * <p/> |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 118 | * <b>Running a single testcase:</b> adb shell am instrument -w |
| 119 | * -e class com.android.foo.FooTest |
| 120 | * com.android.foo/android.test.InstrumentationTestRunner |
| 121 | * <p/> |
| 122 | * <b>Running a single test:</b> adb shell am instrument -w |
| 123 | * -e class com.android.foo.FooTest#testFoo |
| 124 | * com.android.foo/android.test.InstrumentationTestRunner |
| 125 | * <p/> |
| 126 | * <b>Running multiple tests:</b> adb shell am instrument -w |
| 127 | * -e class com.android.foo.FooTest,com.android.foo.TooTest |
| 128 | * com.android.foo/android.test.InstrumentationTestRunner |
| 129 | * <p/> |
Brett Chabot | 89c0ef4 | 2010-03-18 20:03:31 -0700 | [diff] [blame] | 130 | * <b>Running all tests in a java package:</b> adb shell am instrument -w |
| 131 | * -e package com.android.foo.subpkg |
| 132 | * com.android.foo/android.test.InstrumentationTestRunner |
| 133 | * <p/> |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 134 | * <b>Including performance tests:</b> adb shell am instrument -w |
| 135 | * -e perf true |
| 136 | * com.android.foo/android.test.InstrumentationTestRunner |
| 137 | * <p/> |
| 138 | * <b>To debug your tests, set a break point in your code and pass:</b> |
| 139 | * -e debug true |
| 140 | * <p/> |
| 141 | * <b>To run in 'log only' mode</b> |
| 142 | * -e log true |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 143 | * This option will load and iterate through all test classes and methods, but will bypass actual |
| 144 | * test execution. Useful for quickly obtaining info on the tests to be executed by an |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 145 | * instrumentation command. |
| 146 | * <p/> |
| 147 | * <b>To generate EMMA code coverage:</b> |
| 148 | * -e coverage true |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 149 | * Note: this requires an emma instrumented build. By default, the code coverage results file |
Brett Chabot | 51e0364 | 2009-05-28 18:18:15 -0700 | [diff] [blame] | 150 | * will be saved in a /data/<app>/coverage.ec file, unless overridden by coverageFile flag (see |
| 151 | * below) |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 152 | * <p/> |
| 153 | * <b> To specify EMMA code coverage results file path:</b> |
| 154 | * -e coverageFile /sdcard/myFile.ec |
| 155 | * <br/> |
| 156 | * in addition to the other arguments. |
| 157 | */ |
| 158 | |
| 159 | /* (not JavaDoc) |
| 160 | * Although not necessary in most case, another way to use this class is to extend it and have the |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 161 | * derived class return the desired test suite from the {@link #getTestSuite()} method. The test |
| 162 | * suite returned from this method will be used if no target class is defined in the meta-data or |
| 163 | * command line argument parameters. If a derived class is used it needs to be added as an |
| 164 | * instrumentation to the AndroidManifest.xml and the command to run it would look like: |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 165 | * <p/> |
| 166 | * adb shell am instrument -w com.android.foo/<i>com.android.FooInstrumentationTestRunner</i> |
| 167 | * <p/> |
| 168 | * Where <i>com.android.FooInstrumentationTestRunner</i> is the derived class. |
| 169 | * |
| 170 | * This model is used by many existing app tests, but can probably be deprecated. |
| 171 | */ |
| 172 | public class InstrumentationTestRunner extends Instrumentation implements TestSuiteProvider { |
| 173 | |
| 174 | /** @hide */ |
| 175 | public static final String ARGUMENT_TEST_CLASS = "class"; |
| 176 | /** @hide */ |
| 177 | public static final String ARGUMENT_TEST_PACKAGE = "package"; |
| 178 | /** @hide */ |
| 179 | public static final String ARGUMENT_TEST_SIZE_PREDICATE = "size"; |
| 180 | /** @hide */ |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 181 | public static final String ARGUMENT_DELAY_MSEC = "delay_msec"; |
| 182 | |
| 183 | private static final String SMALL_SUITE = "small"; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 184 | private static final String MEDIUM_SUITE = "medium"; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 185 | private static final String LARGE_SUITE = "large"; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 186 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 187 | private static final String ARGUMENT_LOG_ONLY = "log"; |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 188 | /** @hide */ |
| 189 | static final String ARGUMENT_ANNOTATION = "annotation"; |
| 190 | /** @hide */ |
| 191 | static final String ARGUMENT_NOT_ANNOTATION = "notAnnotation"; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 192 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 193 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 194 | * This constant defines the maximum allowed runtime (in ms) for a test included in the "small" |
| 195 | * suite. It is used to make an educated guess at what suite an unlabeled test belongs. |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 196 | */ |
| 197 | private static final float SMALL_SUITE_MAX_RUNTIME = 100; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 198 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 199 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 200 | * This constant defines the maximum allowed runtime (in ms) for a test included in the |
| 201 | * "medium" suite. It is used to make an educated guess at what suite an unlabeled test belongs. |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 202 | */ |
| 203 | private static final float MEDIUM_SUITE_MAX_RUNTIME = 1000; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 204 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 205 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 206 | * The following keys are used in the status bundle to provide structured reports to |
| 207 | * an IInstrumentationWatcher. |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 208 | */ |
| 209 | |
| 210 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 211 | * This value, if stored with key {@link android.app.Instrumentation#REPORT_KEY_IDENTIFIER}, |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 212 | * identifies InstrumentationTestRunner as the source of the report. This is sent with all |
| 213 | * status messages. |
| 214 | */ |
| 215 | public static final String REPORT_VALUE_ID = "InstrumentationTestRunner"; |
| 216 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 217 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 218 | * identifies the total number of tests that are being run. This is sent with all status |
| 219 | * messages. |
| 220 | */ |
| 221 | public static final String REPORT_KEY_NUM_TOTAL = "numtests"; |
| 222 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 223 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 224 | * identifies the sequence number of the current test. This is sent with any status message |
| 225 | * describing a specific test being started or completed. |
| 226 | */ |
| 227 | public static final String REPORT_KEY_NUM_CURRENT = "current"; |
| 228 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 229 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 230 | * identifies the name of the current test class. This is sent with any status message |
| 231 | * describing a specific test being started or completed. |
| 232 | */ |
| 233 | public static final String REPORT_KEY_NAME_CLASS = "class"; |
| 234 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 235 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 236 | * identifies the name of the current test. This is sent with any status message |
| 237 | * describing a specific test being started or completed. |
| 238 | */ |
| 239 | public static final String REPORT_KEY_NAME_TEST = "test"; |
| 240 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 241 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 242 | * reports the run time in seconds of the current test. |
| 243 | */ |
| 244 | private static final String REPORT_KEY_RUN_TIME = "runtime"; |
| 245 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 246 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
Jack Wang | 3fc03e6 | 2010-10-19 15:13:07 -0700 | [diff] [blame] | 247 | * reports the number of total iterations of the current test. |
| 248 | */ |
| 249 | private static final String REPORT_KEY_NUM_ITERATIONS = "numiterations"; |
| 250 | /** |
| 251 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 252 | * reports the guessed suite assignment for the current test. |
| 253 | */ |
| 254 | private static final String REPORT_KEY_SUITE_ASSIGNMENT = "suiteassignment"; |
| 255 | /** |
Brett Chabot | 51e0364 | 2009-05-28 18:18:15 -0700 | [diff] [blame] | 256 | * If included in the status or final bundle sent to an IInstrumentationWatcher, this key |
| 257 | * identifies the path to the generated code coverage file. |
| 258 | */ |
| 259 | private static final String REPORT_KEY_COVERAGE_PATH = "coverageFilePath"; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 260 | |
| 261 | /** |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 262 | * The test is starting. |
| 263 | */ |
| 264 | public static final int REPORT_VALUE_RESULT_START = 1; |
| 265 | /** |
| 266 | * The test completed successfully. |
| 267 | */ |
| 268 | public static final int REPORT_VALUE_RESULT_OK = 0; |
| 269 | /** |
| 270 | * The test completed with an error. |
| 271 | */ |
| 272 | public static final int REPORT_VALUE_RESULT_ERROR = -1; |
| 273 | /** |
| 274 | * The test completed with a failure. |
| 275 | */ |
| 276 | public static final int REPORT_VALUE_RESULT_FAILURE = -2; |
| 277 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 278 | * If included in the status bundle sent to an IInstrumentationWatcher, this key |
| 279 | * identifies a stack trace describing an error or failure. This is sent with any status |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 280 | * message describing a specific test being completed. |
| 281 | */ |
| 282 | public static final String REPORT_KEY_STACK = "stack"; |
| 283 | |
Brett Chabot | 51e0364 | 2009-05-28 18:18:15 -0700 | [diff] [blame] | 284 | // Default file name for code coverage |
| 285 | private static final String DEFAULT_COVERAGE_FILE_NAME = "coverage.ec"; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 286 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 287 | private static final String LOG_TAG = "InstrumentationTestRunner"; |
| 288 | |
| 289 | private final Bundle mResults = new Bundle(); |
David Hu | cbc584d | 2011-09-16 18:08:35 -0700 | [diff] [blame] | 290 | private Bundle mArguments; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 291 | private AndroidTestRunner mTestRunner; |
| 292 | private boolean mDebug; |
| 293 | private boolean mJustCount; |
| 294 | private boolean mSuiteAssignmentMode; |
| 295 | private int mTestCount; |
| 296 | private String mPackageOfTests; |
| 297 | private boolean mCoverage; |
| 298 | private String mCoverageFilePath; |
| 299 | private int mDelayMsec; |
| 300 | |
| 301 | @Override |
| 302 | public void onCreate(Bundle arguments) { |
| 303 | super.onCreate(arguments); |
David Hu | cbc584d | 2011-09-16 18:08:35 -0700 | [diff] [blame] | 304 | mArguments = arguments; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 305 | |
| 306 | // Apk paths used to search for test classes when using TestSuiteBuilders. |
| 307 | String[] apkPaths = |
| 308 | {getTargetContext().getPackageCodePath(), getContext().getPackageCodePath()}; |
| 309 | ClassPathPackageInfoSource.setApkPaths(apkPaths); |
| 310 | |
| 311 | Predicate<TestMethod> testSizePredicate = null; |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 312 | Predicate<TestMethod> testAnnotationPredicate = null; |
| 313 | Predicate<TestMethod> testNotAnnotationPredicate = null; |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 314 | String testClassesArg = null; |
| 315 | boolean logOnly = false; |
| 316 | |
| 317 | if (arguments != null) { |
| 318 | // Test class name passed as an argument should override any meta-data declaration. |
| 319 | testClassesArg = arguments.getString(ARGUMENT_TEST_CLASS); |
| 320 | mDebug = getBooleanArgument(arguments, "debug"); |
| 321 | mJustCount = getBooleanArgument(arguments, "count"); |
| 322 | mSuiteAssignmentMode = getBooleanArgument(arguments, "suiteAssignment"); |
| 323 | mPackageOfTests = arguments.getString(ARGUMENT_TEST_PACKAGE); |
| 324 | testSizePredicate = getSizePredicateFromArg( |
| 325 | arguments.getString(ARGUMENT_TEST_SIZE_PREDICATE)); |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 326 | testAnnotationPredicate = getAnnotationPredicate( |
| 327 | arguments.getString(ARGUMENT_ANNOTATION)); |
| 328 | testNotAnnotationPredicate = getNotAnnotationPredicate( |
| 329 | arguments.getString(ARGUMENT_NOT_ANNOTATION)); |
| 330 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 331 | logOnly = getBooleanArgument(arguments, ARGUMENT_LOG_ONLY); |
| 332 | mCoverage = getBooleanArgument(arguments, "coverage"); |
| 333 | mCoverageFilePath = arguments.getString("coverageFile"); |
| 334 | |
| 335 | try { |
| 336 | Object delay = arguments.get(ARGUMENT_DELAY_MSEC); // Accept either string or int |
| 337 | if (delay != null) mDelayMsec = Integer.parseInt(delay.toString()); |
| 338 | } catch (NumberFormatException e) { |
| 339 | Log.e(LOG_TAG, "Invalid delay_msec parameter", e); |
| 340 | } |
| 341 | } |
| 342 | |
| 343 | TestSuiteBuilder testSuiteBuilder = new TestSuiteBuilder(getClass().getName(), |
| 344 | getTargetContext().getClassLoader()); |
| 345 | |
| 346 | if (testSizePredicate != null) { |
| 347 | testSuiteBuilder.addRequirements(testSizePredicate); |
| 348 | } |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 349 | if (testAnnotationPredicate != null) { |
| 350 | testSuiteBuilder.addRequirements(testAnnotationPredicate); |
| 351 | } |
| 352 | if (testNotAnnotationPredicate != null) { |
| 353 | testSuiteBuilder.addRequirements(testNotAnnotationPredicate); |
| 354 | } |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 355 | |
| 356 | if (testClassesArg == null) { |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 357 | if (mPackageOfTests != null) { |
| 358 | testSuiteBuilder.includePackages(mPackageOfTests); |
| 359 | } else { |
Brett Chabot | 61b10ac | 2009-03-31 17:04:34 -0700 | [diff] [blame] | 360 | TestSuite testSuite = getTestSuite(); |
| 361 | if (testSuite != null) { |
| 362 | testSuiteBuilder.addTestSuite(testSuite); |
| 363 | } else { |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 364 | // no package or class bundle arguments were supplied, and no test suite |
Brett Chabot | 61b10ac | 2009-03-31 17:04:34 -0700 | [diff] [blame] | 365 | // provided so add all tests in application |
| 366 | testSuiteBuilder.includePackages(""); |
| 367 | } |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 368 | } |
| 369 | } else { |
| 370 | parseTestClasses(testClassesArg, testSuiteBuilder); |
| 371 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 372 | |
Urs Grob | da13ef5 | 2009-04-17 11:30:14 -0700 | [diff] [blame] | 373 | testSuiteBuilder.addRequirements(getBuilderRequirements()); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 374 | |
| 375 | mTestRunner = getAndroidTestRunner(); |
| 376 | mTestRunner.setContext(getTargetContext()); |
Jack Wang | 7aba54b | 2009-08-20 19:20:54 -0700 | [diff] [blame] | 377 | mTestRunner.setInstrumentation(this); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 378 | mTestRunner.setSkipExecution(logOnly); |
| 379 | mTestRunner.setTest(testSuiteBuilder.build()); |
| 380 | mTestCount = mTestRunner.getTestCases().size(); |
| 381 | if (mSuiteAssignmentMode) { |
| 382 | mTestRunner.addTestListener(new SuiteAssignmentPrinter()); |
| 383 | } else { |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 384 | WatcherResultPrinter resultPrinter = new WatcherResultPrinter(mTestCount); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 385 | mTestRunner.addTestListener(new TestPrinter("TestRunner", false)); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 386 | mTestRunner.addTestListener(resultPrinter); |
| 387 | mTestRunner.setPerformanceResultsWriter(resultPrinter); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 388 | } |
| 389 | start(); |
| 390 | } |
| 391 | |
David Hu | cbc584d | 2011-09-16 18:08:35 -0700 | [diff] [blame] | 392 | /** |
| 393 | * Get the Bundle object that contains the arguments |
| 394 | * |
| 395 | * @return the Bundle object |
| 396 | * @hide |
| 397 | */ |
| 398 | public Bundle getBundle(){ |
| 399 | return mArguments; |
| 400 | } |
| 401 | |
David Hu | 8cc9a8e | 2011-12-13 15:57:42 -0800 | [diff] [blame] | 402 | /** |
| 403 | * Add a {@link TestListener} |
| 404 | * @hide |
| 405 | */ |
| 406 | protected void addTestListener(TestListener listener){ |
| 407 | if(mTestRunner!=null && listener!=null){ |
| 408 | mTestRunner.addTestListener(listener); |
| 409 | } |
| 410 | } |
| 411 | |
Urs Grob | da13ef5 | 2009-04-17 11:30:14 -0700 | [diff] [blame] | 412 | List<Predicate<TestMethod>> getBuilderRequirements() { |
| 413 | return new ArrayList<Predicate<TestMethod>>(); |
| 414 | } |
| 415 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 416 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 417 | * Parses and loads the specified set of test classes |
| 418 | * |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 419 | * @param testClassArg - comma-separated list of test classes and methods |
| 420 | * @param testSuiteBuilder - builder to add tests to |
| 421 | */ |
| 422 | private void parseTestClasses(String testClassArg, TestSuiteBuilder testSuiteBuilder) { |
| 423 | String[] testClasses = testClassArg.split(","); |
| 424 | for (String testClass : testClasses) { |
| 425 | parseTestClass(testClass, testSuiteBuilder); |
| 426 | } |
| 427 | } |
| 428 | |
| 429 | /** |
| 430 | * Parse and load the given test class and, optionally, method |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 431 | * |
| 432 | * @param testClassName - full package name of test class and optionally method to add. |
| 433 | * Expected format: com.android.TestClass#testMethod |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 434 | * @param testSuiteBuilder - builder to add tests to |
| 435 | */ |
| 436 | private void parseTestClass(String testClassName, TestSuiteBuilder testSuiteBuilder) { |
| 437 | int methodSeparatorIndex = testClassName.indexOf('#'); |
| 438 | String testMethodName = null; |
| 439 | |
| 440 | if (methodSeparatorIndex > 0) { |
| 441 | testMethodName = testClassName.substring(methodSeparatorIndex + 1); |
| 442 | testClassName = testClassName.substring(0, methodSeparatorIndex); |
| 443 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 444 | testSuiteBuilder.addTestClassByName(testClassName, testMethodName, getTargetContext()); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 445 | } |
| 446 | |
| 447 | protected AndroidTestRunner getAndroidTestRunner() { |
| 448 | return new AndroidTestRunner(); |
| 449 | } |
| 450 | |
| 451 | private boolean getBooleanArgument(Bundle arguments, String tag) { |
| 452 | String tagString = arguments.getString(tag); |
| 453 | return tagString != null && Boolean.parseBoolean(tagString); |
| 454 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 455 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 456 | /* |
| 457 | * Returns the size predicate object, corresponding to the "size" argument value. |
| 458 | */ |
| 459 | private Predicate<TestMethod> getSizePredicateFromArg(String sizeArg) { |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 460 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 461 | if (SMALL_SUITE.equals(sizeArg)) { |
| 462 | return TestPredicates.SELECT_SMALL; |
| 463 | } else if (MEDIUM_SUITE.equals(sizeArg)) { |
| 464 | return TestPredicates.SELECT_MEDIUM; |
| 465 | } else if (LARGE_SUITE.equals(sizeArg)) { |
| 466 | return TestPredicates.SELECT_LARGE; |
| 467 | } else { |
| 468 | return null; |
| 469 | } |
| 470 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 471 | |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 472 | /** |
| 473 | * Returns the test predicate object, corresponding to the annotation class value provided via |
| 474 | * the {@link ARGUMENT_ANNOTATION} argument. |
| 475 | * |
| 476 | * @return the predicate or <code>null</code> |
| 477 | */ |
| 478 | private Predicate<TestMethod> getAnnotationPredicate(String annotationClassName) { |
| 479 | Class<? extends Annotation> annotationClass = getAnnotationClass(annotationClassName); |
| 480 | if (annotationClass != null) { |
| 481 | return new HasAnnotation(annotationClass); |
| 482 | } |
| 483 | return null; |
| 484 | } |
| 485 | |
| 486 | /** |
| 487 | * Returns the negative test predicate object, corresponding to the annotation class value |
| 488 | * provided via the {@link ARGUMENT_NOT_ANNOTATION} argument. |
| 489 | * |
| 490 | * @return the predicate or <code>null</code> |
| 491 | */ |
| 492 | private Predicate<TestMethod> getNotAnnotationPredicate(String annotationClassName) { |
| 493 | Class<? extends Annotation> annotationClass = getAnnotationClass(annotationClassName); |
| 494 | if (annotationClass != null) { |
| 495 | return Predicates.not(new HasAnnotation(annotationClass)); |
| 496 | } |
| 497 | return null; |
| 498 | } |
| 499 | |
| 500 | /** |
| 501 | * Helper method to return the annotation class with specified name |
| 502 | * |
| 503 | * @param annotationClassName the fully qualified name of the class |
| 504 | * @return the annotation class or <code>null</code> |
| 505 | */ |
| 506 | private Class<? extends Annotation> getAnnotationClass(String annotationClassName) { |
| 507 | if (annotationClassName == null) { |
| 508 | return null; |
| 509 | } |
| 510 | try { |
| 511 | Class<?> annotationClass = Class.forName(annotationClassName); |
| 512 | if (annotationClass.isAnnotation()) { |
| 513 | return (Class<? extends Annotation>)annotationClass; |
| 514 | } else { |
| 515 | Log.e(LOG_TAG, String.format("Provided annotation value %s is not an Annotation", |
| 516 | annotationClassName)); |
| 517 | } |
| 518 | } catch (ClassNotFoundException e) { |
| 519 | Log.e(LOG_TAG, String.format("Could not find class for specified annotation %s", |
| 520 | annotationClassName)); |
| 521 | } |
| 522 | return null; |
| 523 | } |
| 524 | |
Brett Chabot | 31e7ce7 | 2010-07-07 17:19:08 -0700 | [diff] [blame] | 525 | /** |
| 526 | * Initialize the current thread as a looper. |
| 527 | * <p/> |
| 528 | * Exposed for unit testing. |
| 529 | */ |
| 530 | void prepareLooper() { |
| 531 | Looper.prepare(); |
| 532 | } |
| 533 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 534 | @Override |
| 535 | public void onStart() { |
Brett Chabot | 31e7ce7 | 2010-07-07 17:19:08 -0700 | [diff] [blame] | 536 | prepareLooper(); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 537 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 538 | if (mJustCount) { |
| 539 | mResults.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID); |
| 540 | mResults.putInt(REPORT_KEY_NUM_TOTAL, mTestCount); |
| 541 | finish(Activity.RESULT_OK, mResults); |
| 542 | } else { |
| 543 | if (mDebug) { |
| 544 | Debug.waitForDebugger(); |
| 545 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 546 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 547 | ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); |
| 548 | PrintStream writer = new PrintStream(byteArrayOutputStream); |
| 549 | try { |
| 550 | StringResultPrinter resultPrinter = new StringResultPrinter(writer); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 551 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 552 | mTestRunner.addTestListener(resultPrinter); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 553 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 554 | long startTime = System.currentTimeMillis(); |
| 555 | mTestRunner.runTest(); |
| 556 | long runTime = System.currentTimeMillis() - startTime; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 557 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 558 | resultPrinter.print(mTestRunner.getTestResult(), runTime); |
Brett Chabot | 31e7ce7 | 2010-07-07 17:19:08 -0700 | [diff] [blame] | 559 | } catch (Throwable t) { |
| 560 | // catch all exceptions so a more verbose error message can be outputted |
| 561 | writer.println(String.format("Test run aborted due to unexpected exception: %s", |
| 562 | t.getMessage())); |
| 563 | t.printStackTrace(writer); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 564 | } finally { |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 565 | mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, |
| 566 | String.format("\nTest results for %s=%s", |
| 567 | mTestRunner.getTestClassName(), |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 568 | byteArrayOutputStream.toString())); |
| 569 | |
| 570 | if (mCoverage) { |
| 571 | generateCoverageReport(); |
| 572 | } |
| 573 | writer.close(); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 574 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 575 | finish(Activity.RESULT_OK, mResults); |
| 576 | } |
| 577 | } |
| 578 | } |
| 579 | |
| 580 | public TestSuite getTestSuite() { |
| 581 | return getAllTests(); |
| 582 | } |
| 583 | |
| 584 | /** |
| 585 | * Override this to define all of the tests to run in your package. |
| 586 | */ |
| 587 | public TestSuite getAllTests() { |
| 588 | return null; |
| 589 | } |
| 590 | |
| 591 | /** |
| 592 | * Override this to provide access to the class loader of your package. |
| 593 | */ |
| 594 | public ClassLoader getLoader() { |
| 595 | return null; |
| 596 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 597 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 598 | private void generateCoverageReport() { |
| 599 | // use reflection to call emma dump coverage method, to avoid |
| 600 | // always statically compiling against emma jar |
Brett Chabot | 51e0364 | 2009-05-28 18:18:15 -0700 | [diff] [blame] | 601 | String coverageFilePath = getCoverageFilePath(); |
| 602 | java.io.File coverageFile = new java.io.File(coverageFilePath); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 603 | try { |
Brett Chabot | 88e03a9 | 2010-02-19 09:57:11 -0800 | [diff] [blame] | 604 | Class<?> emmaRTClass = Class.forName("com.vladium.emma.rt.RT"); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 605 | Method dumpCoverageMethod = emmaRTClass.getMethod("dumpCoverageData", |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 606 | coverageFile.getClass(), boolean.class, boolean.class); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 607 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 608 | dumpCoverageMethod.invoke(null, coverageFile, false, false); |
Brett Chabot | 51e0364 | 2009-05-28 18:18:15 -0700 | [diff] [blame] | 609 | // output path to generated coverage file so it can be parsed by a test harness if |
| 610 | // needed |
| 611 | mResults.putString(REPORT_KEY_COVERAGE_PATH, coverageFilePath); |
| 612 | // also output a more user friendly msg |
Brett Chabot | 08d13c3 | 2010-02-18 15:42:13 -0800 | [diff] [blame] | 613 | final String currentStream = mResults.getString( |
| 614 | Instrumentation.REPORT_KEY_STREAMRESULT); |
Brett Chabot | 51e0364 | 2009-05-28 18:18:15 -0700 | [diff] [blame] | 615 | mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, |
Brett Chabot | 08d13c3 | 2010-02-18 15:42:13 -0800 | [diff] [blame] | 616 | String.format("%s\nGenerated code coverage data to %s", currentStream, |
| 617 | coverageFilePath)); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 618 | } catch (ClassNotFoundException e) { |
| 619 | reportEmmaError("Is emma jar on classpath?", e); |
| 620 | } catch (SecurityException e) { |
| 621 | reportEmmaError(e); |
| 622 | } catch (NoSuchMethodException e) { |
| 623 | reportEmmaError(e); |
| 624 | } catch (IllegalArgumentException e) { |
| 625 | reportEmmaError(e); |
| 626 | } catch (IllegalAccessException e) { |
| 627 | reportEmmaError(e); |
| 628 | } catch (InvocationTargetException e) { |
| 629 | reportEmmaError(e); |
| 630 | } |
| 631 | } |
| 632 | |
| 633 | private String getCoverageFilePath() { |
| 634 | if (mCoverageFilePath == null) { |
Brett Chabot | 51e0364 | 2009-05-28 18:18:15 -0700 | [diff] [blame] | 635 | return getTargetContext().getFilesDir().getAbsolutePath() + File.separator + |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 636 | DEFAULT_COVERAGE_FILE_NAME; |
| 637 | } else { |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 638 | return mCoverageFilePath; |
| 639 | } |
| 640 | } |
| 641 | |
| 642 | private void reportEmmaError(Exception e) { |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 643 | reportEmmaError("", e); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 644 | } |
| 645 | |
| 646 | private void reportEmmaError(String hint, Exception e) { |
| 647 | String msg = "Failed to generate emma coverage. " + hint; |
| 648 | Log.e(LOG_TAG, msg, e); |
| 649 | mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "\nError: " + msg); |
| 650 | } |
| 651 | |
| 652 | // TODO kill this, use status() and prettyprint model for better output |
| 653 | private class StringResultPrinter extends ResultPrinter { |
| 654 | |
| 655 | public StringResultPrinter(PrintStream writer) { |
| 656 | super(writer); |
| 657 | } |
| 658 | |
| 659 | synchronized void print(TestResult result, long runTime) { |
| 660 | printHeader(runTime); |
| 661 | printFooter(result); |
| 662 | } |
| 663 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 664 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 665 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 666 | * This class sends status reports back to the IInstrumentationWatcher about |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 667 | * which suite each test belongs. |
| 668 | */ |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 669 | private class SuiteAssignmentPrinter implements TestListener { |
| 670 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 671 | private Bundle mTestResult; |
| 672 | private long mStartTime; |
| 673 | private long mEndTime; |
| 674 | private boolean mTimingValid; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 675 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 676 | public SuiteAssignmentPrinter() { |
| 677 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 678 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 679 | /** |
| 680 | * send a status for the start of a each test, so long tests can be seen as "running" |
| 681 | */ |
| 682 | public void startTest(Test test) { |
| 683 | mTimingValid = true; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 684 | mStartTime = System.currentTimeMillis(); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 685 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 686 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 687 | /** |
| 688 | * @see junit.framework.TestListener#addError(Test, Throwable) |
| 689 | */ |
| 690 | public void addError(Test test, Throwable t) { |
| 691 | mTimingValid = false; |
| 692 | } |
| 693 | |
| 694 | /** |
| 695 | * @see junit.framework.TestListener#addFailure(Test, AssertionFailedError) |
| 696 | */ |
| 697 | public void addFailure(Test test, AssertionFailedError t) { |
| 698 | mTimingValid = false; |
| 699 | } |
| 700 | |
| 701 | /** |
| 702 | * @see junit.framework.TestListener#endTest(Test) |
| 703 | */ |
| 704 | public void endTest(Test test) { |
| 705 | float runTime; |
| 706 | String assignmentSuite; |
| 707 | mEndTime = System.currentTimeMillis(); |
| 708 | mTestResult = new Bundle(); |
| 709 | |
| 710 | if (!mTimingValid || mStartTime < 0) { |
| 711 | assignmentSuite = "NA"; |
| 712 | runTime = -1; |
| 713 | } else { |
| 714 | runTime = mEndTime - mStartTime; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 715 | if (runTime < SMALL_SUITE_MAX_RUNTIME |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 716 | && !InstrumentationTestCase.class.isAssignableFrom(test.getClass())) { |
| 717 | assignmentSuite = SMALL_SUITE; |
| 718 | } else if (runTime < MEDIUM_SUITE_MAX_RUNTIME) { |
| 719 | assignmentSuite = MEDIUM_SUITE; |
| 720 | } else { |
| 721 | assignmentSuite = LARGE_SUITE; |
| 722 | } |
| 723 | } |
| 724 | // Clear mStartTime so that we can verify that it gets set next time. |
| 725 | mStartTime = -1; |
| 726 | |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 727 | mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, |
| 728 | test.getClass().getName() + "#" + ((TestCase) test).getName() |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 729 | + "\nin " + assignmentSuite + " suite\nrunTime: " |
| 730 | + String.valueOf(runTime) + "\n"); |
| 731 | mTestResult.putFloat(REPORT_KEY_RUN_TIME, runTime); |
| 732 | mTestResult.putString(REPORT_KEY_SUITE_ASSIGNMENT, assignmentSuite); |
| 733 | |
| 734 | sendStatus(0, mTestResult); |
| 735 | } |
| 736 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 737 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 738 | /** |
| 739 | * This class sends status reports back to the IInstrumentationWatcher |
| 740 | */ |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 741 | private class WatcherResultPrinter implements TestListener, PerformanceResultsWriter { |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 742 | private final Bundle mResultTemplate; |
| 743 | Bundle mTestResult; |
| 744 | int mTestNum = 0; |
| 745 | int mTestResultCode = 0; |
| 746 | String mTestClass = null; |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 747 | PerformanceCollector mPerfCollector = new PerformanceCollector(); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 748 | boolean mIsTimedTest = false; |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 749 | boolean mIncludeDetailedStats = false; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 750 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 751 | public WatcherResultPrinter(int numTests) { |
| 752 | mResultTemplate = new Bundle(); |
| 753 | mResultTemplate.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID); |
| 754 | mResultTemplate.putInt(REPORT_KEY_NUM_TOTAL, numTests); |
| 755 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 756 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 757 | /** |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 758 | * send a status for the start of a each test, so long tests can be seen |
| 759 | * as "running" |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 760 | */ |
| 761 | public void startTest(Test test) { |
| 762 | String testClass = test.getClass().getName(); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 763 | String testName = ((TestCase)test).getName(); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 764 | mTestResult = new Bundle(mResultTemplate); |
| 765 | mTestResult.putString(REPORT_KEY_NAME_CLASS, testClass); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 766 | mTestResult.putString(REPORT_KEY_NAME_TEST, testName); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 767 | mTestResult.putInt(REPORT_KEY_NUM_CURRENT, ++mTestNum); |
| 768 | // pretty printing |
| 769 | if (testClass != null && !testClass.equals(mTestClass)) { |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 770 | mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 771 | String.format("\n%s:", testClass)); |
| 772 | mTestClass = testClass; |
| 773 | } else { |
| 774 | mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, ""); |
| 775 | } |
| 776 | |
Jack Wang | 3fc03e6 | 2010-10-19 15:13:07 -0700 | [diff] [blame] | 777 | Method testMethod = null; |
| 778 | try { |
| 779 | testMethod = test.getClass().getMethod(testName); |
| 780 | // Report total number of iterations, if test is repetitive |
| 781 | if (testMethod.isAnnotationPresent(RepetitiveTest.class)) { |
| 782 | int numIterations = testMethod.getAnnotation( |
| 783 | RepetitiveTest.class).numIterations(); |
| 784 | mTestResult.putInt(REPORT_KEY_NUM_ITERATIONS, numIterations); |
| 785 | } |
| 786 | } catch (NoSuchMethodException e) { |
| 787 | // ignore- the test with given name does not exist. Will be handled during test |
| 788 | // execution |
| 789 | } |
| 790 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 791 | // The delay_msec parameter is normally used to provide buffers of idle time |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 792 | // for power measurement purposes. To make sure there is a delay before and after |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 793 | // every test in a suite, we delay *after* every test (see endTest below) and also |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 794 | // delay *before* the first test. So, delay test1 delay test2 delay. |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 795 | |
| 796 | try { |
| 797 | if (mTestNum == 1) Thread.sleep(mDelayMsec); |
| 798 | } catch (InterruptedException e) { |
| 799 | throw new IllegalStateException(e); |
| 800 | } |
| 801 | |
| 802 | sendStatus(REPORT_VALUE_RESULT_START, mTestResult); |
| 803 | mTestResultCode = 0; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 804 | |
| 805 | mIsTimedTest = false; |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 806 | mIncludeDetailedStats = false; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 807 | try { |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 808 | // Look for TimedTest annotation on both test class and test method |
Brian Muramatsu | 4898395 | 2010-11-17 12:28:51 -0800 | [diff] [blame] | 809 | if (testMethod != null && testMethod.isAnnotationPresent(TimedTest.class)) { |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 810 | mIsTimedTest = true; |
Jack Wang | 3fc03e6 | 2010-10-19 15:13:07 -0700 | [diff] [blame] | 811 | mIncludeDetailedStats = testMethod.getAnnotation( |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 812 | TimedTest.class).includeDetailedStats(); |
| 813 | } else if (test.getClass().isAnnotationPresent(TimedTest.class)) { |
| 814 | mIsTimedTest = true; |
| 815 | mIncludeDetailedStats = test.getClass().getAnnotation( |
| 816 | TimedTest.class).includeDetailedStats(); |
| 817 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 818 | } catch (SecurityException e) { |
Brett Chabot | 31e7ce7 | 2010-07-07 17:19:08 -0700 | [diff] [blame] | 819 | // ignore - the test with given name cannot be accessed. Will be handled during |
| 820 | // test execution |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 821 | } |
| 822 | |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 823 | if (mIsTimedTest && mIncludeDetailedStats) { |
| 824 | mPerfCollector.beginSnapshot(""); |
| 825 | } else if (mIsTimedTest) { |
| 826 | mPerfCollector.startTiming(""); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 827 | } |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 828 | } |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 829 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 830 | /** |
| 831 | * @see junit.framework.TestListener#addError(Test, Throwable) |
| 832 | */ |
| 833 | public void addError(Test test, Throwable t) { |
| 834 | mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t)); |
| 835 | mTestResultCode = REPORT_VALUE_RESULT_ERROR; |
| 836 | // pretty printing |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 837 | mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, |
| 838 | String.format("\nError in %s:\n%s", |
| 839 | ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t))); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 840 | } |
| 841 | |
| 842 | /** |
| 843 | * @see junit.framework.TestListener#addFailure(Test, AssertionFailedError) |
| 844 | */ |
| 845 | public void addFailure(Test test, AssertionFailedError t) { |
| 846 | mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t)); |
| 847 | mTestResultCode = REPORT_VALUE_RESULT_FAILURE; |
| 848 | // pretty printing |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 849 | mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, |
| 850 | String.format("\nFailure in %s:\n%s", |
| 851 | ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t))); |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 852 | } |
| 853 | |
| 854 | /** |
| 855 | * @see junit.framework.TestListener#endTest(Test) |
| 856 | */ |
| 857 | public void endTest(Test test) { |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 858 | if (mIsTimedTest && mIncludeDetailedStats) { |
| 859 | mTestResult.putAll(mPerfCollector.endSnapshot()); |
| 860 | } else if (mIsTimedTest) { |
| 861 | writeStopTiming(mPerfCollector.stopTiming("")); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 862 | } |
| 863 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 864 | if (mTestResultCode == 0) { |
| 865 | mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "."); |
| 866 | } |
| 867 | sendStatus(mTestResultCode, mTestResult); |
| 868 | |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 869 | try { // Sleep after every test, if specified |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 870 | Thread.sleep(mDelayMsec); |
| 871 | } catch (InterruptedException e) { |
| 872 | throw new IllegalStateException(e); |
| 873 | } |
| 874 | } |
| 875 | |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 876 | public void writeBeginSnapshot(String label) { |
| 877 | // Do nothing |
| 878 | } |
| 879 | |
| 880 | public void writeEndSnapshot(Bundle results) { |
Jack Wang | 075997f | 2009-10-27 22:01:09 -0700 | [diff] [blame] | 881 | // Copy all snapshot data fields into mResults, which is outputted |
| 882 | // via Instrumentation.finish |
| 883 | mResults.putAll(results); |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 884 | } |
| 885 | |
| 886 | public void writeStartTiming(String label) { |
| 887 | // Do nothing |
| 888 | } |
| 889 | |
| 890 | public void writeStopTiming(Bundle results) { |
| 891 | // Copy results into mTestResult by flattening list of iterations, |
| 892 | // which is outputted via WatcherResultPrinter.endTest |
| 893 | int i = 0; |
| 894 | for (Parcelable p : |
| 895 | results.getParcelableArrayList(PerformanceCollector.METRIC_KEY_ITERATIONS)) { |
| 896 | Bundle iteration = (Bundle)p; |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 897 | String index = "iteration" + i + "."; |
Jack Wang | ff1df69 | 2009-08-26 17:19:13 -0700 | [diff] [blame] | 898 | mTestResult.putString(index + PerformanceCollector.METRIC_KEY_LABEL, |
| 899 | iteration.getString(PerformanceCollector.METRIC_KEY_LABEL)); |
| 900 | mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_CPU_TIME, |
| 901 | iteration.getLong(PerformanceCollector.METRIC_KEY_CPU_TIME)); |
| 902 | mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_EXECUTION_TIME, |
| 903 | iteration.getLong(PerformanceCollector.METRIC_KEY_EXECUTION_TIME)); |
| 904 | i++; |
| 905 | } |
| 906 | } |
| 907 | |
Jack Wang | 075997f | 2009-10-27 22:01:09 -0700 | [diff] [blame] | 908 | public void writeMeasurement(String label, long value) { |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 909 | mTestResult.putLong(label, value); |
Jack Wang | 075997f | 2009-10-27 22:01:09 -0700 | [diff] [blame] | 910 | } |
| 911 | |
| 912 | public void writeMeasurement(String label, float value) { |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 913 | mTestResult.putFloat(label, value); |
Jack Wang | 075997f | 2009-10-27 22:01:09 -0700 | [diff] [blame] | 914 | } |
| 915 | |
| 916 | public void writeMeasurement(String label, String value) { |
Jack Wang | 4f414bd | 2009-11-06 20:53:47 -0800 | [diff] [blame] | 917 | mTestResult.putString(label, value); |
Jack Wang | 075997f | 2009-10-27 22:01:09 -0700 | [diff] [blame] | 918 | } |
| 919 | |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 920 | // TODO report the end of the cycle |
The Android Open Source Project | 9066cfe | 2009-03-03 19:31:44 -0800 | [diff] [blame] | 921 | } |
| 922 | } |