Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2010 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package com.android.cts.tradefed.testtype; |
| 18 | |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 19 | import com.android.cts.tradefed.build.CtsBuildHelper; |
Brett Chabot | 56ac786 | 2010-10-10 15:14:51 -0700 | [diff] [blame] | 20 | import com.android.cts.tradefed.device.DeviceInfoCollector; |
Brett Chabot | 817c153 | 2011-10-20 14:40:43 -0700 | [diff] [blame] | 21 | import com.android.cts.tradefed.result.CtsTestStatus; |
| 22 | import com.android.cts.tradefed.result.PlanCreator; |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 23 | import com.android.ddmlib.Log; |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 24 | import com.android.ddmlib.Log.LogLevel; |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 25 | import com.android.ddmlib.testrunner.TestIdentifier; |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 26 | import com.android.tradefed.build.IBuildInfo; |
Brett Chabot | 817c153 | 2011-10-20 14:40:43 -0700 | [diff] [blame] | 27 | import com.android.tradefed.config.ConfigurationException; |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 28 | import com.android.tradefed.config.Option; |
Brett Chabot | 7e54460 | 2011-06-20 14:48:25 -0700 | [diff] [blame] | 29 | import com.android.tradefed.config.Option.Importance; |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 30 | import com.android.tradefed.device.DeviceNotAvailableException; |
| 31 | import com.android.tradefed.device.ITestDevice; |
Keun young Park | 4bbacc1 | 2012-07-11 16:02:41 -0700 | [diff] [blame] | 32 | import com.android.tradefed.device.TestDeviceOptions; |
Brett Chabot | da6997c | 2011-10-18 14:34:41 -0700 | [diff] [blame] | 33 | import com.android.tradefed.log.LogUtil.CLog; |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 34 | import com.android.tradefed.result.ITestInvocationListener; |
Brett Chabot | 7e54460 | 2011-06-20 14:48:25 -0700 | [diff] [blame] | 35 | import com.android.tradefed.result.InputStreamSource; |
| 36 | import com.android.tradefed.result.LogDataType; |
Brett Chabot | f8b02e4 | 2011-10-28 12:48:43 -0700 | [diff] [blame] | 37 | import com.android.tradefed.result.ResultForwarder; |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 38 | import com.android.tradefed.testtype.IBuildReceiver; |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 39 | import com.android.tradefed.testtype.IDeviceTest; |
| 40 | import com.android.tradefed.testtype.IRemoteTest; |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 41 | import com.android.tradefed.testtype.IResumableTest; |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 42 | import com.android.tradefed.testtype.IShardableTest; |
Brett Chabot | e7aa76d | 2013-02-20 15:19:30 -0800 | [diff] [blame] | 43 | import com.android.tradefed.util.RunUtil; |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 44 | import com.android.tradefed.util.xml.AbstractXmlParser.ParseException; |
| 45 | |
Brett Chabot | 1217b9d | 2013-10-31 16:51:34 -0700 | [diff] [blame] | 46 | import junit.framework.Test; |
| 47 | |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 48 | import java.io.BufferedInputStream; |
| 49 | import java.io.File; |
| 50 | import java.io.FileInputStream; |
| 51 | import java.io.FileNotFoundException; |
| 52 | import java.io.InputStream; |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 53 | import java.util.ArrayList; |
Keun young Park | 9011422 | 2012-08-14 20:43:54 -0700 | [diff] [blame] | 54 | import java.util.Arrays; |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 55 | import java.util.Collection; |
Brett Chabot | 58c43a8 | 2011-09-13 14:13:57 -0700 | [diff] [blame] | 56 | import java.util.HashMap; |
Brett Chabot | 358dc56 | 2011-10-25 15:46:48 -0700 | [diff] [blame] | 57 | import java.util.HashSet; |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 58 | import java.util.LinkedHashSet; |
| 59 | import java.util.LinkedList; |
| 60 | import java.util.List; |
Brett Chabot | 58c43a8 | 2011-09-13 14:13:57 -0700 | [diff] [blame] | 61 | import java.util.Map; |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 62 | import java.util.Queue; |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 63 | import java.util.Set; |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 64 | |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 65 | /** |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 66 | * A {@link Test} for running CTS tests. |
| 67 | * <p/> |
| 68 | * Supports running all the tests contained in a CTS plan, or individual test packages. |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 69 | */ |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 70 | public class CtsTest implements IDeviceTest, IResumableTest, IShardableTest, IBuildReceiver { |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 71 | private static final String LOG_TAG = "CtsTest"; |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 72 | |
Brett Chabot | 51b9caa | 2011-09-07 18:44:22 -0700 | [diff] [blame] | 73 | public static final String PLAN_OPTION = "plan"; |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 74 | private static final String PACKAGE_OPTION = "package"; |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 75 | private static final String CLASS_OPTION = "class"; |
| 76 | private static final String METHOD_OPTION = "method"; |
Brett Chabot | 817c153 | 2011-10-20 14:40:43 -0700 | [diff] [blame] | 77 | public static final String CONTINUE_OPTION = "continue-session"; |
Brian Muramatsu | 066fb28 | 2012-01-13 14:09:19 -0800 | [diff] [blame] | 78 | public static final String RUN_KNOWN_FAILURES_OPTION = "run-known-failures"; |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 79 | |
Brett Chabot | 58c43a8 | 2011-09-13 14:13:57 -0700 | [diff] [blame] | 80 | public static final String PACKAGE_NAME_METRIC = "packageName"; |
| 81 | public static final String PACKAGE_DIGEST_METRIC = "packageDigest"; |
| 82 | |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 83 | private ITestDevice mDevice; |
| 84 | |
Brett Chabot | 7e54460 | 2011-06-20 14:48:25 -0700 | [diff] [blame] | 85 | @Option(name = PLAN_OPTION, description = "the test plan to run.", |
| 86 | importance = Importance.IF_UNSET) |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 87 | private String mPlanName = null; |
| 88 | |
Brett Chabot | 58c43a8 | 2011-09-13 14:13:57 -0700 | [diff] [blame] | 89 | @Option(name = PACKAGE_OPTION, shortName = 'p', description = "the test packages(s) to run.", |
Brett Chabot | 7e54460 | 2011-06-20 14:48:25 -0700 | [diff] [blame] | 90 | importance = Importance.IF_UNSET) |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 91 | private Collection<String> mPackageNames = new ArrayList<String>(); |
| 92 | |
Brett Chabot | 7e54460 | 2011-06-20 14:48:25 -0700 | [diff] [blame] | 93 | @Option(name = "exclude-package", description = "the test packages(s) to exclude from the run.") |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 94 | private Collection<String> mExcludedPackageNames = new ArrayList<String>(); |
| 95 | |
Brett Chabot | 7e54460 | 2011-06-20 14:48:25 -0700 | [diff] [blame] | 96 | @Option(name = CLASS_OPTION, shortName = 'c', description = "run a specific test class.", |
| 97 | importance = Importance.IF_UNSET) |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 98 | private String mClassName = null; |
| 99 | |
| 100 | @Option(name = METHOD_OPTION, shortName = 'm', |
Brett Chabot | 7e54460 | 2011-06-20 14:48:25 -0700 | [diff] [blame] | 101 | description = "run a specific test method, from given --class.", |
| 102 | importance = Importance.IF_UNSET) |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 103 | private String mMethodName = null; |
| 104 | |
Brett Chabot | 817c153 | 2011-10-20 14:40:43 -0700 | [diff] [blame] | 105 | @Option(name = CONTINUE_OPTION, |
| 106 | description = "continue a previous test session.", |
| 107 | importance = Importance.IF_UNSET) |
| 108 | private Integer mContinueSessionId = null; |
| 109 | |
Brett Chabot | bd3e510 | 2011-10-25 17:19:22 -0700 | [diff] [blame] | 110 | @Option(name = "skip-device-info", shortName = 'd', description = |
| 111 | "flag to control whether to collect info from device. Providing this flag will speed up " + |
| 112 | "test execution for short test runs but will result in required data being omitted from " + |
| 113 | "the test report.") |
| 114 | private boolean mSkipDeviceInfo = false; |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 115 | |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 116 | @Option(name = "resume", description = |
Brett Chabot | 7e54460 | 2011-06-20 14:48:25 -0700 | [diff] [blame] | 117 | "flag to attempt to automatically resume aborted test run on another connected device. ") |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 118 | private boolean mResume = false; |
| 119 | |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 120 | @Option(name = "shards", description = |
| 121 | "shard the tests to run into separately runnable chunks to execute on multiple devices " + |
Brett Chabot | 7e54460 | 2011-06-20 14:48:25 -0700 | [diff] [blame] | 122 | "concurrently.") |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 123 | private int mShards = 1; |
| 124 | |
Brett Chabot | 7e54460 | 2011-06-20 14:48:25 -0700 | [diff] [blame] | 125 | @Option(name = "screenshot", description = |
| 126 | "flag for taking a screenshot of the device when test execution is complete.") |
| 127 | private boolean mScreenshot = false; |
| 128 | |
Brett Chabot | f8b02e4 | 2011-10-28 12:48:43 -0700 | [diff] [blame] | 129 | @Option(name = "bugreport", shortName = 'b', description = |
| 130 | "take a bugreport after each failed test. " + |
| 131 | "Warning: can potentially use a lot of disk space.") |
| 132 | private boolean mBugreport = false; |
| 133 | |
Brian Muramatsu | 066fb28 | 2012-01-13 14:09:19 -0800 | [diff] [blame] | 134 | @Option(name = RUN_KNOWN_FAILURES_OPTION, shortName = 'k', description = |
| 135 | "run tests including known failures") |
| 136 | private boolean mIncludeKnownFailures; |
| 137 | |
Keun young Park | b749e15 | 2012-10-08 14:52:02 -0700 | [diff] [blame] | 138 | @Option(name = "disable-reboot", description = |
| 139 | "Do not reboot device after running some amount of tests. Default behavior is to reboot.") |
| 140 | private boolean mDisableReboot = false; |
Keun young Park | e39a70f | 2012-07-11 12:16:38 -0700 | [diff] [blame] | 141 | |
Keun young Park | d5527a3 | 2012-07-12 15:29:57 -0700 | [diff] [blame] | 142 | @Option(name = "reboot-wait-time", description = |
Keun young Park | b749e15 | 2012-10-08 14:52:02 -0700 | [diff] [blame] | 143 | "Additional wait time in ms after boot complete.") |
Keun young Park | 495620e | 2012-07-17 10:30:54 -0700 | [diff] [blame] | 144 | private int mRebootWaitTimeMSec = 2 * 60 * 1000; |
Keun young Park | d5527a3 | 2012-07-12 15:29:57 -0700 | [diff] [blame] | 145 | |
| 146 | @Option(name = "reboot-interval", description = |
Keun young Park | b749e15 | 2012-10-08 14:52:02 -0700 | [diff] [blame] | 147 | "Interval between each reboot in min.") |
Keun young Park | d5527a3 | 2012-07-12 15:29:57 -0700 | [diff] [blame] | 148 | private int mRebootIntervalMin = 30; |
| 149 | |
Brett Chabot | f428b3d | 2013-05-16 22:03:20 -0700 | [diff] [blame] | 150 | @Option(name = "screenshot-on-failure", description = |
| 151 | "take a screenshot on every test failure.") |
| 152 | private boolean mScreenshotOnFailures = false; |
Keun young Park | 9011422 | 2012-08-14 20:43:54 -0700 | [diff] [blame] | 153 | |
Brett Chabot | e7aa76d | 2013-02-20 15:19:30 -0800 | [diff] [blame] | 154 | @Option(name = "logcat-on-failure", description = |
| 155 | "take a logcat snapshot on every test failure. Unlike --bugreport, this can capture" + |
| 156 | "logs even if connection with device has been lost, as well as being much more " + |
| 157 | "performant.") |
| 158 | private boolean mLogcatOnFailures = false; |
| 159 | |
| 160 | @Option(name = "logcat-on-failure-size", description = |
| 161 | "The max number of logcat data in bytes to capture when --logcat-on-failure is on. " + |
| 162 | "Should be an amount that can comfortably fit in memory.") |
| 163 | private int mMaxLogcatBytes = 500 * 1024; // 500K |
| 164 | |
Keun young Park | 9011422 | 2012-08-14 20:43:54 -0700 | [diff] [blame] | 165 | private long mPrevRebootTime; // last reboot time |
| 166 | |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 167 | /** data structure for a {@link IRemoteTest} and its known tests */ |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 168 | class TestPackage { |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 169 | private final IRemoteTest mTestForPackage; |
| 170 | private final Collection<TestIdentifier> mKnownTests; |
Brett Chabot | 58c43a8 | 2011-09-13 14:13:57 -0700 | [diff] [blame] | 171 | private final ITestPackageDef mPackageDef; |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 172 | |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 173 | TestPackage(ITestPackageDef packageDef, IRemoteTest testForPackage, |
Brett Chabot | 58c43a8 | 2011-09-13 14:13:57 -0700 | [diff] [blame] | 174 | Collection<TestIdentifier> knownTests) { |
| 175 | mPackageDef = packageDef; |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 176 | mTestForPackage = testForPackage; |
| 177 | mKnownTests = knownTests; |
| 178 | } |
| 179 | |
| 180 | IRemoteTest getTestForPackage() { |
| 181 | return mTestForPackage; |
| 182 | } |
| 183 | |
| 184 | Collection<TestIdentifier> getKnownTests() { |
| 185 | return mKnownTests; |
| 186 | } |
Brett Chabot | 58c43a8 | 2011-09-13 14:13:57 -0700 | [diff] [blame] | 187 | |
| 188 | ITestPackageDef getPackageDef() { |
| 189 | return mPackageDef; |
| 190 | } |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 191 | |
| 192 | /** |
| 193 | * Return the test run name that should be used for the TestPackage |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 194 | */ |
| 195 | String getTestRunName() { |
| 196 | return mPackageDef.getUri(); |
| 197 | } |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 198 | } |
| 199 | |
Brett Chabot | f8b02e4 | 2011-10-28 12:48:43 -0700 | [diff] [blame] | 200 | /** |
| 201 | * A {@link ResultForwarder} that will forward a bugreport on each failed test. |
| 202 | */ |
| 203 | private static class FailedTestBugreportGenerator extends ResultForwarder { |
| 204 | private ITestDevice mDevice; |
| 205 | |
| 206 | public FailedTestBugreportGenerator(ITestInvocationListener listener, ITestDevice device) { |
| 207 | super(listener); |
| 208 | mDevice = device; |
| 209 | } |
| 210 | |
| 211 | @Override |
| 212 | public void testFailed(TestFailure status, TestIdentifier test, String trace) { |
| 213 | super.testFailed(status, test, trace); |
| 214 | InputStreamSource bugSource = mDevice.getBugreport(); |
Brett Chabot | e92f07b | 2013-02-12 10:42:05 -0800 | [diff] [blame] | 215 | super.testLog(String.format("bug-%s_%s", test.getClassName(), test.getTestName()), |
| 216 | LogDataType.TEXT, bugSource); |
Brett Chabot | f8b02e4 | 2011-10-28 12:48:43 -0700 | [diff] [blame] | 217 | bugSource.cancel(); |
| 218 | } |
| 219 | } |
| 220 | |
Brett Chabot | e92f07b | 2013-02-12 10:42:05 -0800 | [diff] [blame] | 221 | /** |
Brett Chabot | e7aa76d | 2013-02-20 15:19:30 -0800 | [diff] [blame] | 222 | * A {@link ResultForwarder} that will forward a logcat snapshot on each failed test. |
| 223 | */ |
| 224 | private static class FailedTestLogcatGenerator extends ResultForwarder { |
| 225 | private ITestDevice mDevice; |
| 226 | private int mNumLogcatBytes; |
| 227 | |
| 228 | public FailedTestLogcatGenerator(ITestInvocationListener listener, ITestDevice device, |
| 229 | int maxLogcatBytes) { |
| 230 | super(listener); |
| 231 | mDevice = device; |
| 232 | mNumLogcatBytes = maxLogcatBytes; |
| 233 | } |
| 234 | |
| 235 | @Override |
| 236 | public void testFailed(TestFailure status, TestIdentifier test, String trace) { |
| 237 | super.testFailed(status, test, trace); |
| 238 | // sleep a small amount of time to ensure test failure stack trace makes it into logcat |
| 239 | // capture |
| 240 | RunUtil.getDefault().sleep(10); |
| 241 | InputStreamSource logSource = mDevice.getLogcat(mNumLogcatBytes); |
| 242 | super.testLog(String.format("logcat-%s_%s", test.getClassName(), test.getTestName()), |
| 243 | LogDataType.TEXT, logSource); |
| 244 | logSource.cancel(); |
| 245 | } |
| 246 | } |
| 247 | |
| 248 | /** |
Brett Chabot | f428b3d | 2013-05-16 22:03:20 -0700 | [diff] [blame] | 249 | * A {@link ResultForwarder} that will forward a screenshot on test failures. |
Brett Chabot | e92f07b | 2013-02-12 10:42:05 -0800 | [diff] [blame] | 250 | */ |
Brett Chabot | f428b3d | 2013-05-16 22:03:20 -0700 | [diff] [blame] | 251 | private static class FailedTestScreenshotGenerator extends ResultForwarder { |
Brett Chabot | e92f07b | 2013-02-12 10:42:05 -0800 | [diff] [blame] | 252 | private ITestDevice mDevice; |
| 253 | |
Brett Chabot | f428b3d | 2013-05-16 22:03:20 -0700 | [diff] [blame] | 254 | public FailedTestScreenshotGenerator(ITestInvocationListener listener, |
Brett Chabot | e92f07b | 2013-02-12 10:42:05 -0800 | [diff] [blame] | 255 | ITestDevice device) { |
| 256 | super(listener); |
| 257 | mDevice = device; |
| 258 | } |
| 259 | |
| 260 | @Override |
| 261 | public void testFailed(TestFailure status, TestIdentifier test, String trace) { |
| 262 | super.testFailed(status, test, trace); |
| 263 | |
Brett Chabot | f428b3d | 2013-05-16 22:03:20 -0700 | [diff] [blame] | 264 | try { |
| 265 | InputStreamSource screenSource = mDevice.getScreenshot(); |
| 266 | super.testLog(String.format("screenshot-%s_%s", test.getClassName(), |
| 267 | test.getTestName()), LogDataType.PNG, screenSource); |
| 268 | screenSource.cancel(); |
| 269 | } catch (DeviceNotAvailableException e) { |
| 270 | // TODO: rethrow this somehow |
| 271 | CLog.e("Device %s became unavailable while capturing screenshot, %s", |
| 272 | mDevice.getSerialNumber(), e.toString()); |
Brett Chabot | e92f07b | 2013-02-12 10:42:05 -0800 | [diff] [blame] | 273 | } |
| 274 | } |
| 275 | } |
| 276 | |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 277 | /** list of remaining tests to execute */ |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 278 | private List<TestPackage> mRemainingTestPkgs = null; |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 279 | |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 280 | private CtsBuildHelper mCtsBuild = null; |
Brett Chabot | 4263db4 | 2011-04-15 13:51:48 -0700 | [diff] [blame] | 281 | private IBuildInfo mBuildInfo = null; |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 282 | |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 283 | /** |
| 284 | * {@inheritDoc} |
| 285 | */ |
Brian Muramatsu | c1ca42f | 2011-12-07 11:12:42 -0800 | [diff] [blame] | 286 | @Override |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 287 | public ITestDevice getDevice() { |
| 288 | return mDevice; |
| 289 | } |
| 290 | |
| 291 | /** |
| 292 | * {@inheritDoc} |
| 293 | */ |
Brian Muramatsu | c1ca42f | 2011-12-07 11:12:42 -0800 | [diff] [blame] | 294 | @Override |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 295 | public void setDevice(ITestDevice device) { |
| 296 | mDevice = device; |
| 297 | } |
| 298 | |
| 299 | /** |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 300 | * Set the plan name to run. |
| 301 | * <p/> |
| 302 | * Exposed for unit testing |
| 303 | */ |
| 304 | void setPlanName(String planName) { |
| 305 | mPlanName = planName; |
| 306 | } |
| 307 | |
| 308 | /** |
Brett Chabot | bd3e510 | 2011-10-25 17:19:22 -0700 | [diff] [blame] | 309 | * Set the skip collect device info flag. |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 310 | * <p/> |
| 311 | * Exposed for unit testing |
| 312 | */ |
Brett Chabot | bd3e510 | 2011-10-25 17:19:22 -0700 | [diff] [blame] | 313 | void setSkipDeviceInfo(boolean skipDeviceInfo) { |
| 314 | mSkipDeviceInfo = skipDeviceInfo; |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 315 | } |
| 316 | |
| 317 | /** |
| 318 | * Adds a package name to the list of test packages to run. |
| 319 | * <p/> |
| 320 | * Exposed for unit testing |
| 321 | */ |
| 322 | void addPackageName(String packageName) { |
| 323 | mPackageNames.add(packageName); |
| 324 | } |
| 325 | |
| 326 | /** |
| 327 | * Adds a package name to the list of test packages to exclude. |
| 328 | * <p/> |
| 329 | * Exposed for unit testing |
| 330 | */ |
| 331 | void addExcludedPackageName(String packageName) { |
| 332 | mExcludedPackageNames.add(packageName); |
| 333 | } |
| 334 | |
| 335 | /** |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 336 | * Set the test class name to run. |
| 337 | * <p/> |
| 338 | * Exposed for unit testing |
| 339 | */ |
| 340 | void setClassName(String className) { |
| 341 | mClassName = className; |
| 342 | } |
| 343 | |
| 344 | /** |
| 345 | * Set the test method name to run. |
| 346 | * <p/> |
| 347 | * Exposed for unit testing |
| 348 | */ |
| 349 | void setMethodName(String methodName) { |
| 350 | mMethodName = methodName; |
| 351 | } |
| 352 | |
| 353 | /** |
Brett Chabot | 817c153 | 2011-10-20 14:40:43 -0700 | [diff] [blame] | 354 | * Sets the test session id to continue. |
| 355 | * <p/> |
| 356 | * Exposed for unit testing |
| 357 | */ |
| 358 | void setContinueSessionId(int sessionId) { |
| 359 | mContinueSessionId = sessionId; |
| 360 | } |
| 361 | |
| 362 | /** |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 363 | * {@inheritDoc} |
| 364 | */ |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 365 | @Override |
| 366 | public boolean isResumable() { |
| 367 | return mResume; |
| 368 | } |
| 369 | |
| 370 | /** |
| 371 | * {@inheritDoc} |
| 372 | */ |
| 373 | @Override |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 374 | public void setBuild(IBuildInfo build) { |
Brett Chabot | f4bec73 | 2011-04-19 17:31:06 -0700 | [diff] [blame] | 375 | mCtsBuild = CtsBuildHelper.createBuildHelper(build); |
| 376 | mBuildInfo = build; |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 377 | } |
| 378 | |
| 379 | /** |
| 380 | * Set the CTS build container. |
| 381 | * <p/> |
| 382 | * Exposed so unit tests can mock the provided build. |
| 383 | * |
| 384 | * @param buildHelper |
| 385 | */ |
| 386 | void setBuildHelper(CtsBuildHelper buildHelper) { |
| 387 | mCtsBuild = buildHelper; |
| 388 | } |
| 389 | |
| 390 | /** |
| 391 | * {@inheritDoc} |
| 392 | */ |
| 393 | @Override |
Brett Chabot | 7cc0e25 | 2011-02-10 09:56:45 -0800 | [diff] [blame] | 394 | public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 395 | if (getDevice() == null) { |
| 396 | throw new IllegalArgumentException("missing device"); |
| 397 | } |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 398 | |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 399 | if (mRemainingTestPkgs == null) { |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 400 | checkFields(); |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 401 | mRemainingTestPkgs = buildTestsToRun(); |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 402 | } |
Brett Chabot | f8b02e4 | 2011-10-28 12:48:43 -0700 | [diff] [blame] | 403 | if (mBugreport) { |
| 404 | FailedTestBugreportGenerator bugListener = new FailedTestBugreportGenerator(listener, |
| 405 | getDevice()); |
| 406 | listener = bugListener; |
| 407 | } |
Brett Chabot | f428b3d | 2013-05-16 22:03:20 -0700 | [diff] [blame] | 408 | if (mScreenshotOnFailures) { |
| 409 | FailedTestScreenshotGenerator screenListener = new FailedTestScreenshotGenerator( |
Brett Chabot | e92f07b | 2013-02-12 10:42:05 -0800 | [diff] [blame] | 410 | listener, getDevice()); |
| 411 | listener = screenListener; |
| 412 | } |
Brett Chabot | e7aa76d | 2013-02-20 15:19:30 -0800 | [diff] [blame] | 413 | if (mLogcatOnFailures) { |
| 414 | FailedTestLogcatGenerator logcatListener = new FailedTestLogcatGenerator( |
| 415 | listener, getDevice(), mMaxLogcatBytes); |
| 416 | listener = logcatListener; |
| 417 | } |
Brett Chabot | 7e54460 | 2011-06-20 14:48:25 -0700 | [diff] [blame] | 418 | |
Brett Chabot | 358dc56 | 2011-10-25 15:46:48 -0700 | [diff] [blame] | 419 | // collect and install the prerequisiteApks first, to save time when multiple test |
| 420 | // packages are using the same prerequisite apk (I'm looking at you, CtsTestStubs!) |
| 421 | Collection<String> prerequisiteApks = getPrerequisiteApks(mRemainingTestPkgs); |
| 422 | Collection<String> uninstallPackages = getPrerequisitePackageNames(mRemainingTestPkgs); |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 423 | ResultFilter filter = new ResultFilter(listener, mRemainingTestPkgs); |
| 424 | |
| 425 | try { |
Brett Chabot | 358dc56 | 2011-10-25 15:46:48 -0700 | [diff] [blame] | 426 | installPrerequisiteApks(prerequisiteApks); |
Keun young Park | 282188d | 2012-07-23 13:31:35 -0700 | [diff] [blame] | 427 | |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 428 | // always collect the device info, even for resumed runs, since test will likely be |
| 429 | // running on a different device |
| 430 | collectDeviceInfo(getDevice(), mCtsBuild, listener); |
Keun young Park | b749e15 | 2012-10-08 14:52:02 -0700 | [diff] [blame] | 431 | if (mRemainingTestPkgs.size() > 1 && !mDisableReboot) { |
Keun young Park | 495620e | 2012-07-17 10:30:54 -0700 | [diff] [blame] | 432 | Log.i(LOG_TAG, "Initial reboot for multiple packages"); |
| 433 | rebootDevice(); |
| 434 | } |
Keun young Park | 9011422 | 2012-08-14 20:43:54 -0700 | [diff] [blame] | 435 | mPrevRebootTime = System.currentTimeMillis(); |
| 436 | |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 437 | while (!mRemainingTestPkgs.isEmpty()) { |
| 438 | TestPackage knownTests = mRemainingTestPkgs.get(0); |
| 439 | |
| 440 | IRemoteTest test = knownTests.getTestForPackage(); |
| 441 | if (test instanceof IDeviceTest) { |
| 442 | ((IDeviceTest)test).setDevice(getDevice()); |
| 443 | } |
| 444 | if (test instanceof IBuildReceiver) { |
| 445 | ((IBuildReceiver)test).setBuild(mBuildInfo); |
| 446 | } |
| 447 | |
| 448 | forwardPackageDetails(knownTests.getPackageDef(), listener); |
| 449 | test.run(filter); |
| 450 | mRemainingTestPkgs.remove(0); |
Keun young Park | c323c37 | 2012-07-20 13:54:46 -0700 | [diff] [blame] | 451 | if (mRemainingTestPkgs.size() > 0) { |
Keun young Park | 9011422 | 2012-08-14 20:43:54 -0700 | [diff] [blame] | 452 | rebootIfNecessary(knownTests, mRemainingTestPkgs.get(0)); |
Keun young Park | c323c37 | 2012-07-20 13:54:46 -0700 | [diff] [blame] | 453 | // remove artifacts like status bar from the previous test. |
| 454 | // But this cannot dismiss dialog popped-up. |
| 455 | changeToHomeScreen(); |
Keun young Park | e39a70f | 2012-07-11 12:16:38 -0700 | [diff] [blame] | 456 | } |
Brett Chabot | 7e54460 | 2011-06-20 14:48:25 -0700 | [diff] [blame] | 457 | } |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 458 | |
| 459 | if (mScreenshot) { |
| 460 | InputStreamSource screenshotSource = getDevice().getScreenshot(); |
| 461 | try { |
| 462 | listener.testLog("screenshot", LogDataType.PNG, screenshotSource); |
| 463 | } finally { |
| 464 | screenshotSource.cancel(); |
| 465 | } |
| 466 | } |
Brett Chabot | 358dc56 | 2011-10-25 15:46:48 -0700 | [diff] [blame] | 467 | |
| 468 | uninstallPrequisiteApks(uninstallPackages); |
| 469 | |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 470 | } finally { |
| 471 | filter.reportUnexecutedTests(); |
Brett Chabot | 7e54460 | 2011-06-20 14:48:25 -0700 | [diff] [blame] | 472 | } |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 473 | } |
| 474 | |
Keun young Park | 9011422 | 2012-08-14 20:43:54 -0700 | [diff] [blame] | 475 | private void rebootIfNecessary(TestPackage testFinished, TestPackage testToRun) |
| 476 | throws DeviceNotAvailableException { |
| 477 | // If there comes spurious failure like INJECT_EVENTS for a package, |
| 478 | // reboot it before running it. |
| 479 | // Also reboot after package which is know to leave pop-up behind |
Keun young Park | 8a3df62 | 2012-11-12 13:33:15 -0800 | [diff] [blame] | 480 | final List<String> rebootAfterList = Arrays.asList( |
| 481 | "CtsMediaTestCases", |
| 482 | "CtsAccessibilityTestCases"); |
| 483 | final List<String> rebootBeforeList = Arrays.asList( |
| 484 | "CtsAnimationTestCases", |
Keun young Park | 9011422 | 2012-08-14 20:43:54 -0700 | [diff] [blame] | 485 | "CtsGraphicsTestCases", |
| 486 | "CtsViewTestCases", |
Keun young Park | 9011422 | 2012-08-14 20:43:54 -0700 | [diff] [blame] | 487 | "CtsWidgetTestCases" ); |
| 488 | long intervalInMSec = mRebootIntervalMin * 60 * 1000; |
keunyoung | 5dc9df7 | 2013-04-01 16:05:59 -0700 | [diff] [blame] | 489 | if (mDevice.getSerialNumber().startsWith("emulator-")) { |
| 490 | return; |
| 491 | } |
Keun young Park | b749e15 | 2012-10-08 14:52:02 -0700 | [diff] [blame] | 492 | if (!mDisableReboot) { |
Keun young Park | 9011422 | 2012-08-14 20:43:54 -0700 | [diff] [blame] | 493 | long currentTime = System.currentTimeMillis(); |
| 494 | if (((currentTime - mPrevRebootTime) > intervalInMSec) || |
| 495 | rebootAfterList.contains(testFinished.getPackageDef().getName()) || |
| 496 | rebootBeforeList.contains(testToRun.getPackageDef().getName()) ) { |
| 497 | Log.i(LOG_TAG, |
| 498 | String.format("Rebooting after running package %s, before package %s", |
| 499 | testFinished.getPackageDef().getName(), |
| 500 | testToRun.getPackageDef().getName())); |
| 501 | rebootDevice(); |
| 502 | mPrevRebootTime = System.currentTimeMillis(); |
| 503 | } |
| 504 | } |
| 505 | } |
| 506 | |
Keun young Park | 495620e | 2012-07-17 10:30:54 -0700 | [diff] [blame] | 507 | private void rebootDevice() throws DeviceNotAvailableException { |
Keun young Park | 2aee05a | 2013-02-15 18:29:58 -0800 | [diff] [blame] | 508 | final int TIMEOUT_MS = 10 * 60 * 1000; |
Keun young Park | 495620e | 2012-07-17 10:30:54 -0700 | [diff] [blame] | 509 | TestDeviceOptions options = mDevice.getOptions(); |
| 510 | // store default value and increase time-out for reboot |
| 511 | int rebootTimeout = options.getRebootTimeout(); |
| 512 | long onlineTimeout = options.getOnlineTimeout(); |
| 513 | options.setRebootTimeout(TIMEOUT_MS); |
| 514 | options.setOnlineTimeout(TIMEOUT_MS); |
| 515 | mDevice.setOptions(options); |
| 516 | |
| 517 | mDevice.reboot(); |
| 518 | |
| 519 | // restore default values |
| 520 | options.setRebootTimeout(rebootTimeout); |
| 521 | options.setOnlineTimeout(onlineTimeout); |
| 522 | mDevice.setOptions(options); |
| 523 | Log.i(LOG_TAG, "Rebooting done"); |
| 524 | try { |
| 525 | Thread.sleep(mRebootWaitTimeMSec); |
| 526 | } catch (InterruptedException e) { |
| 527 | Log.i(LOG_TAG, "Boot wait interrupted"); |
| 528 | } |
| 529 | } |
| 530 | |
Keun young Park | c323c37 | 2012-07-20 13:54:46 -0700 | [diff] [blame] | 531 | private void changeToHomeScreen() throws DeviceNotAvailableException { |
| 532 | final String homeCmd = "input keyevent 3"; |
| 533 | |
| 534 | mDevice.executeShellCommand(homeCmd); |
| 535 | try { |
| 536 | Thread.sleep(1000); |
| 537 | } catch (InterruptedException e) { |
| 538 | //ignore |
| 539 | } |
| 540 | } |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 541 | /** |
| 542 | * Build the list of test packages to run |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 543 | */ |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 544 | private List<TestPackage> buildTestsToRun() { |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 545 | List<TestPackage> testPkgList = new LinkedList<TestPackage>(); |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 546 | try { |
Brett Chabot | b791837 | 2011-10-19 19:03:44 -0700 | [diff] [blame] | 547 | ITestPackageRepo testRepo = createTestCaseRepo(); |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 548 | Collection<ITestPackageDef> testPkgDefs = getTestPackagesToRun(testRepo); |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 549 | |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 550 | for (ITestPackageDef testPkgDef : testPkgDefs) { |
| 551 | addTestPackage(testPkgList, testPkgDef); |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 552 | } |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 553 | if (testPkgList.isEmpty()) { |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 554 | Log.logAndDisplay(LogLevel.WARN, LOG_TAG, "No tests to run"); |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 555 | } |
| 556 | } catch (FileNotFoundException e) { |
| 557 | throw new IllegalArgumentException("failed to find CTS plan file", e); |
| 558 | } catch (ParseException e) { |
| 559 | throw new IllegalArgumentException("failed to parse CTS plan file", e); |
Brett Chabot | 817c153 | 2011-10-20 14:40:43 -0700 | [diff] [blame] | 560 | } catch (ConfigurationException e) { |
| 561 | throw new IllegalArgumentException("failed to process arguments", e); |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 562 | } |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 563 | return testPkgList; |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 564 | } |
| 565 | |
| 566 | /** |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 567 | * Adds a test package to the list of packages to test |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 568 | * |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 569 | * @param testList |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 570 | * @param testPkgDef |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 571 | */ |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 572 | private void addTestPackage(List<TestPackage> testList, ITestPackageDef testPkgDef) { |
| 573 | IRemoteTest testForPackage = testPkgDef.createTest(mCtsBuild.getTestCasesDir()); |
| 574 | if (testForPackage != null) { |
| 575 | Collection<TestIdentifier> knownTests = testPkgDef.getTests(); |
| 576 | testList.add(new TestPackage(testPkgDef, testForPackage, knownTests)); |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 577 | } |
| 578 | } |
| 579 | |
| 580 | /** |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 581 | * Return the list of test package defs to run |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 582 | * |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 583 | * @return the list of test package defs to run |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 584 | * @throws ParseException |
| 585 | * @throws FileNotFoundException |
Brett Chabot | 817c153 | 2011-10-20 14:40:43 -0700 | [diff] [blame] | 586 | * @throws ConfigurationException |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 587 | */ |
Brett Chabot | b791837 | 2011-10-19 19:03:44 -0700 | [diff] [blame] | 588 | private Collection<ITestPackageDef> getTestPackagesToRun(ITestPackageRepo testRepo) |
Brett Chabot | 817c153 | 2011-10-20 14:40:43 -0700 | [diff] [blame] | 589 | throws ParseException, FileNotFoundException, ConfigurationException { |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 590 | // use LinkedHashSet to have predictable iteration order |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 591 | Set<ITestPackageDef> testPkgDefs = new LinkedHashSet<ITestPackageDef>(); |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 592 | if (mPlanName != null) { |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 593 | Log.i(LOG_TAG, String.format("Executing CTS test plan %s", mPlanName)); |
Brett Chabot | b791837 | 2011-10-19 19:03:44 -0700 | [diff] [blame] | 594 | File ctsPlanFile = mCtsBuild.getTestPlanFile(mPlanName); |
Brett Chabot | 817c153 | 2011-10-20 14:40:43 -0700 | [diff] [blame] | 595 | ITestPlan plan = createPlan(mPlanName); |
| 596 | plan.parse(createXmlStream(ctsPlanFile)); |
| 597 | for (String uri : plan.getTestUris()) { |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 598 | if (!mExcludedPackageNames.contains(uri)) { |
| 599 | ITestPackageDef testPackage = testRepo.getTestPackage(uri); |
Brett Chabot | 1217b9d | 2013-10-31 16:51:34 -0700 | [diff] [blame] | 600 | if (testPackage != null) { |
| 601 | testPackage.setExcludedTestFilter(plan.getExcludedTestFilter(uri)); |
| 602 | testPkgDefs.add(testPackage); |
| 603 | } else { |
| 604 | CLog.e("Could not find test package uri %s referenced in plan %s", uri, |
| 605 | mPlanName); |
| 606 | } |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 607 | } |
| 608 | } |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 609 | } else if (mPackageNames.size() > 0){ |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 610 | Log.i(LOG_TAG, String.format("Executing CTS test packages %s", mPackageNames)); |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 611 | for (String uri : mPackageNames) { |
| 612 | ITestPackageDef testPackage = testRepo.getTestPackage(uri); |
| 613 | if (testPackage != null) { |
| 614 | testPkgDefs.add(testPackage); |
| 615 | } else { |
| 616 | throw new IllegalArgumentException(String.format( |
| 617 | "Could not find test package %s. " + |
| 618 | "Use 'list packages' to see available packages." , uri)); |
| 619 | } |
| 620 | } |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 621 | } else if (mClassName != null) { |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 622 | Log.i(LOG_TAG, String.format("Executing CTS test class %s", mClassName)); |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 623 | // try to find package to run from class name |
| 624 | String packageUri = testRepo.findPackageForTest(mClassName); |
| 625 | if (packageUri != null) { |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 626 | ITestPackageDef testPackageDef = testRepo.getTestPackage(packageUri); |
| 627 | testPackageDef.setClassName(mClassName, mMethodName); |
| 628 | testPkgDefs.add(testPackageDef); |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 629 | } else { |
| 630 | Log.logAndDisplay(LogLevel.WARN, LOG_TAG, String.format( |
| 631 | "Could not find package for test class %s", mClassName)); |
| 632 | } |
Brett Chabot | 817c153 | 2011-10-20 14:40:43 -0700 | [diff] [blame] | 633 | } else if (mContinueSessionId != null) { |
| 634 | // create an in-memory derived plan that contains the notExecuted tests from previous |
| 635 | // session |
| 636 | // use timestamp as plan name so it will hopefully be unique |
| 637 | String uniquePlanName = Long.toString(System.currentTimeMillis()); |
| 638 | PlanCreator planCreator = new PlanCreator(uniquePlanName, mContinueSessionId, |
| 639 | CtsTestStatus.NOT_EXECUTED); |
| 640 | ITestPlan plan = createPlan(planCreator); |
| 641 | for (String uri : plan.getTestUris()) { |
| 642 | if (!mExcludedPackageNames.contains(uri)) { |
| 643 | ITestPackageDef testPackage = testRepo.getTestPackage(uri); |
| 644 | testPackage.setExcludedTestFilter(plan.getExcludedTestFilter(uri)); |
| 645 | testPkgDefs.add(testPackage); |
| 646 | } |
| 647 | } |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 648 | } else { |
| 649 | // should never get here - was checkFields() not called? |
| 650 | throw new IllegalStateException("nothing to run?"); |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 651 | } |
Brett Chabot | 53e68a3 | 2011-10-05 14:14:55 -0700 | [diff] [blame] | 652 | return testPkgDefs; |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 653 | } |
| 654 | |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 655 | /** |
Brett Chabot | 358dc56 | 2011-10-25 15:46:48 -0700 | [diff] [blame] | 656 | * Return the list of unique prerequisite Android package names |
| 657 | * @param testPackages |
Brett Chabot | 358dc56 | 2011-10-25 15:46:48 -0700 | [diff] [blame] | 658 | */ |
| 659 | private Collection<String> getPrerequisitePackageNames(List<TestPackage> testPackages) { |
| 660 | Set<String> pkgNames = new HashSet<String>(); |
| 661 | for (TestPackage testPkg : testPackages) { |
| 662 | String pkgName = testPkg.mPackageDef.getTargetPackageName(); |
| 663 | if (pkgName != null) { |
| 664 | pkgNames.add(pkgName); |
| 665 | } |
| 666 | } |
| 667 | return pkgNames; |
| 668 | } |
| 669 | |
| 670 | /** |
| 671 | * Return the list of unique prerequisite apks to install |
| 672 | * @param testPackages |
Brett Chabot | 358dc56 | 2011-10-25 15:46:48 -0700 | [diff] [blame] | 673 | */ |
| 674 | private Collection<String> getPrerequisiteApks(List<TestPackage> testPackages) { |
| 675 | Set<String> apkNames = new HashSet<String>(); |
| 676 | for (TestPackage testPkg : testPackages) { |
| 677 | String apkName = testPkg.mPackageDef.getTargetApkName(); |
| 678 | if (apkName != null) { |
| 679 | apkNames.add(apkName); |
| 680 | } |
| 681 | } |
| 682 | return apkNames; |
| 683 | } |
| 684 | |
| 685 | /** |
| 686 | * Install the collection of test apk file names |
| 687 | * |
| 688 | * @param prerequisiteApks |
| 689 | * @throws DeviceNotAvailableException |
| 690 | */ |
| 691 | private void installPrerequisiteApks(Collection<String> prerequisiteApks) |
| 692 | throws DeviceNotAvailableException { |
| 693 | for (String apkName : prerequisiteApks) { |
| 694 | try { |
| 695 | File apkFile = mCtsBuild.getTestApp(apkName); |
| 696 | String errorCode = getDevice().installPackage(apkFile, true); |
| 697 | if (errorCode != null) { |
| 698 | CLog.e("Failed to install %s. Reason: %s", apkName, errorCode); |
| 699 | } |
| 700 | } catch (FileNotFoundException e) { |
| 701 | CLog.e("Could not find test apk %s", apkName); |
| 702 | } |
| 703 | } |
| 704 | } |
| 705 | |
| 706 | /** |
| 707 | * Uninstalls the collection of android package names from device. |
| 708 | * |
| 709 | * @param uninstallPackages |
| 710 | */ |
| 711 | private void uninstallPrequisiteApks(Collection<String> uninstallPackages) |
| 712 | throws DeviceNotAvailableException { |
| 713 | for (String pkgName : uninstallPackages) { |
| 714 | getDevice().uninstallPackage(pkgName); |
| 715 | } |
| 716 | } |
| 717 | |
| 718 | /** |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 719 | * {@inheritDoc} |
| 720 | */ |
| 721 | @Override |
| 722 | public Collection<IRemoteTest> split() { |
| 723 | if (mShards <= 1) { |
| 724 | return null; |
| 725 | } |
| 726 | checkFields(); |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 727 | List<TestPackage> allTests = buildTestsToRun(); |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 728 | |
| 729 | if (allTests.size() <= 1) { |
| 730 | Log.w(LOG_TAG, "no tests to shard!"); |
| 731 | return null; |
| 732 | } |
| 733 | |
| 734 | // treat shardQueue as a circular queue, to sequentially distribute tests among shards |
| 735 | Queue<IRemoteTest> shardQueue = new LinkedList<IRemoteTest>(); |
| 736 | // don't create more shards than the number of tests we have! |
| 737 | for (int i = 0; i < mShards && i < allTests.size(); i++) { |
| 738 | CtsTest shard = new CtsTest(); |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 739 | shard.mRemainingTestPkgs = new LinkedList<TestPackage>(); |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 740 | shardQueue.add(shard); |
| 741 | } |
| 742 | while (!allTests.isEmpty()) { |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 743 | TestPackage testPair = allTests.remove(0); |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 744 | CtsTest shard = (CtsTest)shardQueue.poll(); |
Brett Chabot | cbf4ad2 | 2011-09-28 10:39:55 -0700 | [diff] [blame] | 745 | shard.mRemainingTestPkgs.add(testPair); |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 746 | shardQueue.add(shard); |
| 747 | } |
| 748 | return shardQueue; |
| 749 | } |
| 750 | |
| 751 | /** |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 752 | * Runs the device info collector instrumentation on device, and forwards it to test listeners |
| 753 | * as run metrics. |
| 754 | * <p/> |
| 755 | * Exposed so unit tests can mock. |
| 756 | * |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 757 | * @throws DeviceNotAvailableException |
| 758 | */ |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 759 | void collectDeviceInfo(ITestDevice device, CtsBuildHelper ctsBuild, |
| 760 | ITestInvocationListener listener) throws DeviceNotAvailableException { |
Brett Chabot | bd3e510 | 2011-10-25 17:19:22 -0700 | [diff] [blame] | 761 | if (!mSkipDeviceInfo) { |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 762 | DeviceInfoCollector.collectDeviceInfo(device, ctsBuild.getTestCasesDir(), listener); |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 763 | } |
| 764 | } |
| 765 | |
| 766 | /** |
Brett Chabot | b791837 | 2011-10-19 19:03:44 -0700 | [diff] [blame] | 767 | * Factory method for creating a {@link ITestPackageRepo}. |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 768 | * <p/> |
| 769 | * Exposed for unit testing |
| 770 | */ |
Brett Chabot | b791837 | 2011-10-19 19:03:44 -0700 | [diff] [blame] | 771 | ITestPackageRepo createTestCaseRepo() { |
Brian Muramatsu | 066fb28 | 2012-01-13 14:09:19 -0800 | [diff] [blame] | 772 | return new TestPackageRepo(mCtsBuild.getTestCasesDir(), mIncludeKnownFailures); |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 773 | } |
| 774 | |
| 775 | /** |
Brett Chabot | fad0a7c | 2011-10-17 15:30:28 -0700 | [diff] [blame] | 776 | * Factory method for creating a {@link TestPlan}. |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 777 | * <p/> |
| 778 | * Exposed for unit testing |
| 779 | */ |
Brett Chabot | b791837 | 2011-10-19 19:03:44 -0700 | [diff] [blame] | 780 | ITestPlan createPlan(String planName) { |
| 781 | return new TestPlan(planName); |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 782 | } |
| 783 | |
| 784 | /** |
Brett Chabot | 817c153 | 2011-10-20 14:40:43 -0700 | [diff] [blame] | 785 | * Factory method for creating a {@link TestPlan} from a {@link PlanCreator}. |
| 786 | * <p/> |
| 787 | * Exposed for unit testing |
| 788 | * @throws ConfigurationException |
| 789 | */ |
| 790 | ITestPlan createPlan(PlanCreator planCreator) throws ConfigurationException { |
| 791 | return planCreator.createDerivedPlan(mCtsBuild); |
| 792 | } |
| 793 | |
| 794 | /** |
Brett Chabot | 1dcb9a5 | 2011-02-10 20:26:57 -0800 | [diff] [blame] | 795 | * Factory method for creating a {@link InputStream} from a plan xml file. |
| 796 | * <p/> |
| 797 | * Exposed for unit testing |
| 798 | */ |
| 799 | InputStream createXmlStream(File xmlFile) throws FileNotFoundException { |
| 800 | return new BufferedInputStream(new FileInputStream(xmlFile)); |
| 801 | } |
| 802 | |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 803 | private void checkFields() { |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 804 | // for simplicity of command line usage, make --plan, --package, and --class mutually |
| 805 | // exclusive |
| 806 | boolean mutualExclusiveArgs = xor(mPlanName != null, mPackageNames.size() > 0, |
Brett Chabot | 817c153 | 2011-10-20 14:40:43 -0700 | [diff] [blame] | 807 | mClassName != null, mContinueSessionId != null); |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 808 | |
| 809 | if (!mutualExclusiveArgs) { |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 810 | throw new IllegalArgumentException(String.format( |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 811 | "Ambiguous or missing arguments. " + |
Brett Chabot | f8b02e4 | 2011-10-28 12:48:43 -0700 | [diff] [blame] | 812 | "One and only one of --%s --%s(s), --%s or --%s to run can be specified", |
Brett Chabot | 817c153 | 2011-10-20 14:40:43 -0700 | [diff] [blame] | 813 | PLAN_OPTION, PACKAGE_OPTION, CLASS_OPTION, CONTINUE_OPTION)); |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 814 | } |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 815 | if (mMethodName != null && mClassName == null) { |
Brett Chabot | 0fb89bf | 2011-01-14 12:17:19 -0800 | [diff] [blame] | 816 | throw new IllegalArgumentException(String.format( |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 817 | "Must specify --%s when --%s is used", CLASS_OPTION, METHOD_OPTION)); |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 818 | } |
Brett Chabot | 3fc406b | 2011-02-14 15:55:00 -0800 | [diff] [blame] | 819 | if (mCtsBuild == null) { |
| 820 | throw new IllegalArgumentException("missing CTS build"); |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 821 | } |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 822 | } |
| 823 | |
| 824 | /** |
Brett Chabot | ef5a604 | 2011-01-19 19:54:14 -0800 | [diff] [blame] | 825 | * Helper method to perform exclusive or on list of boolean arguments |
| 826 | * |
| 827 | * @param args set of booleans on which to perform exclusive or |
| 828 | * @return <code>true</code> if one and only one of <var>args</code> is <code>true</code>. |
| 829 | * Otherwise return <code>false</code>. |
| 830 | */ |
| 831 | private boolean xor(boolean... args) { |
| 832 | boolean currentVal = args[0]; |
| 833 | for (int i=1; i < args.length; i++) { |
| 834 | if (currentVal && args[i]) { |
| 835 | return false; |
| 836 | } |
| 837 | currentVal |= args[i]; |
| 838 | } |
| 839 | return currentVal; |
| 840 | } |
Brett Chabot | 58c43a8 | 2011-09-13 14:13:57 -0700 | [diff] [blame] | 841 | |
| 842 | /** |
| 843 | * Forward the digest and package name to the listener as a metric |
| 844 | * |
| 845 | * @param listener |
| 846 | */ |
| 847 | private void forwardPackageDetails(ITestPackageDef def, ITestInvocationListener listener) { |
| 848 | Map<String, String> metrics = new HashMap<String, String>(2); |
| 849 | metrics.put(PACKAGE_NAME_METRIC, def.getName()); |
| 850 | metrics.put(PACKAGE_DIGEST_METRIC, def.getDigest()); |
| 851 | listener.testRunStarted(def.getUri(), 0); |
| 852 | listener.testRunEnded(0, metrics); |
| 853 | } |
Brett Chabot | 3add916 | 2010-09-12 17:23:05 -0700 | [diff] [blame] | 854 | } |