Merge "Report unexecuted expected instrumentation tests"
diff --git a/invocation_interfaces/com/android/tradefed/result/TestRunResult.java b/invocation_interfaces/com/android/tradefed/result/TestRunResult.java
index 91e934e..234ef2b 100644
--- a/invocation_interfaces/com/android/tradefed/result/TestRunResult.java
+++ b/invocation_interfaces/com/android/tradefed/result/TestRunResult.java
@@ -298,6 +298,10 @@
updateTestResult(test, TestStatus.FAILURE, FailureDescription.create(trace));
}
+ public void testFailed(TestDescription test, FailureDescription failure) {
+ updateTestResult(test, TestStatus.FAILURE, failure);
+ }
+
public void testAssumptionFailure(TestDescription test, String trace) {
updateTestResult(test, TestStatus.ASSUMPTION_FAILURE, FailureDescription.create(trace));
}
diff --git a/src/com/android/tradefed/result/CollectingTestListener.java b/src/com/android/tradefed/result/CollectingTestListener.java
index 7f46ee1..e12902a 100644
--- a/src/com/android/tradefed/result/CollectingTestListener.java
+++ b/src/com/android/tradefed/result/CollectingTestListener.java
@@ -298,6 +298,12 @@
}
@Override
+ public void testFailed(TestDescription test, FailureDescription failure) {
+ setCountDirty();
+ mCurrentTestRunResult.testFailed(test, failure);
+ }
+
+ @Override
public void testAssumptionFailure(TestDescription test, String trace) {
setCountDirty();
mCurrentTestRunResult.testAssumptionFailure(test, trace);
diff --git a/test_framework/com/android/tradefed/testtype/InstrumentationFileTest.java b/test_framework/com/android/tradefed/testtype/InstrumentationFileTest.java
index 5fbbd76..7c0d7b6 100644
--- a/test_framework/com/android/tradefed/testtype/InstrumentationFileTest.java
+++ b/test_framework/com/android/tradefed/testtype/InstrumentationFileTest.java
@@ -203,7 +203,7 @@
} finally {
deleteTestFileFromDevice(mFilePathOnDevice);
Collection<TestDescription> completedTests =
- testTracker.getCurrentRunResults().getCompletedTests();
+ InstrumentationTest.excludeNonExecuted(testTracker.getCurrentRunResults());
if (mTests.removeAll(completedTests) && !mTests.isEmpty()) {
// re-run remaining tests from file
writeTestsToFileAndRun(mTests, testInfo, listener);
diff --git a/test_framework/com/android/tradefed/testtype/InstrumentationListener.java b/test_framework/com/android/tradefed/testtype/InstrumentationListener.java
index 2a96294..baafb76 100644
--- a/test_framework/com/android/tradefed/testtype/InstrumentationListener.java
+++ b/test_framework/com/android/tradefed/testtype/InstrumentationListener.java
@@ -45,6 +45,7 @@
private Set<TestDescription> mDuplicateTests = new HashSet<>();
private final Collection<TestDescription> mExpectedTests;
private boolean mDisableDuplicateCheck = false;
+ private boolean mReportUnexecutedTests = false;
private ProcessInfo mSystemServerProcess = null;
/**
@@ -68,6 +69,10 @@
mSystemServerProcess = info;
}
+ public void setReportUnexecutedTests(boolean enable) {
+ mReportUnexecutedTests = enable;
+ }
+
@Override
public void testRunStarted(String runName, int testCount) {
// In case of crash, run will attempt to report with 0
@@ -123,6 +128,18 @@
mDuplicateTests));
error.setFailureStatus(FailureStatus.TEST_FAILURE);
super.testRunFailed(error);
+ } else if (mReportUnexecutedTests && mExpectedTests.size() > mTests.size()) {
+ Set<TestDescription> missingTests = new LinkedHashSet<>(mExpectedTests);
+ missingTests.removeAll(mTests);
+ for (TestDescription miss : missingTests) {
+ super.testStarted(miss);
+ FailureDescription failure =
+ FailureDescription.create(
+ "test did not run due to instrumentation issue.",
+ FailureStatus.NOT_EXECUTED);
+ super.testFailed(miss, failure);
+ super.testEnded(miss, new HashMap<String, Metric>());
+ }
}
super.testRunEnded(elapsedTime, runMetrics);
}
diff --git a/test_framework/com/android/tradefed/testtype/InstrumentationTest.java b/test_framework/com/android/tradefed/testtype/InstrumentationTest.java
index 97ae0dd..108f920 100644
--- a/test_framework/com/android/tradefed/testtype/InstrumentationTest.java
+++ b/test_framework/com/android/tradefed/testtype/InstrumentationTest.java
@@ -45,8 +45,10 @@
import com.android.tradefed.result.CollectingTestListener;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.result.TestDescription;
+import com.android.tradefed.result.TestResult;
import com.android.tradefed.result.TestRunResult;
import com.android.tradefed.result.ddmlib.DefaultRemoteAndroidTestRunner;
+import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
import com.android.tradefed.retry.IRetryDecision;
import com.android.tradefed.retry.RetryStrategy;
import com.android.tradefed.testtype.coverage.CoverageOptions;
@@ -68,6 +70,7 @@
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -315,6 +318,12 @@
+ "a system_server restart.")
private boolean mEnableSoftRestartCheck = false;
+ @Option(
+ name = "report-unexecuted-tests",
+ description =
+ "Whether or not to enable reporting all unexecuted tests from instrumentation.")
+ private boolean mReportUnexecuted = true;
+
private IAbi mAbi = null;
private Collection<String> mInstallArgs = new ArrayList<>();
@@ -1053,13 +1062,14 @@
instrumentationListener.setOriginalSystemServer(
getDevice().getProcessByName("system_server"));
}
+ instrumentationListener.setReportUnexecutedTests(mReportUnexecuted);
mDevice.runInstrumentationTests(mRunner, instrumentationListener);
TestRunResult testRun = testTracker.getCurrentRunResults();
if (testRun.isRunFailure() || !testRun.getCompletedTests().containsAll(expectedTests)) {
// Don't re-run any completed tests, unless this is a coverage run.
if (mConfiguration != null
&& !mConfiguration.getCoverageOptions().isCoverageEnabled()) {
- expectedTests.removeAll(testTracker.getCurrentRunResults().getCompletedTests());
+ expectedTests.removeAll(excludeNonExecuted(testTracker.getCurrentRunResults()));
IRetryDecision decision = mConfiguration.getRetryDecision();
if (!RetryStrategy.NO_RETRY.equals(decision.getRetryStrategy())
&& decision.getMaxRetryCount() > 1) {
@@ -1073,6 +1083,20 @@
}
}
+ /** Filter out "NOT_EXECUTED" for the purpose of tracking what needs to be rerun. */
+ protected static Set<TestDescription> excludeNonExecuted(TestRunResult results) {
+ Set<TestDescription> completedTest = results.getCompletedTests();
+ for (Entry<TestDescription, TestResult> entry : results.getTestResults().entrySet()) {
+ if (completedTest.contains(entry.getKey()) && entry.getValue().getFailure() != null) {
+ if (FailureStatus.NOT_EXECUTED.equals(
+ entry.getValue().getFailure().getFailureStatus())) {
+ completedTest.remove(entry.getKey());
+ }
+ }
+ }
+ return completedTest;
+ }
+
/**
* Rerun any <var>mRemainingTests</var>
*
diff --git a/tests/src/com/android/tradefed/testtype/InstrumentationFileTestTest.java b/tests/src/com/android/tradefed/testtype/InstrumentationFileTestTest.java
index 09c5ef4..46e4b7b 100644
--- a/tests/src/com/android/tradefed/testtype/InstrumentationFileTestTest.java
+++ b/tests/src/com/android/tradefed/testtype/InstrumentationFileTestTest.java
@@ -31,6 +31,7 @@
import com.android.tradefed.invoker.InvocationContext;
import com.android.tradefed.invoker.TestInformation;
import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
+import com.android.tradefed.result.FailureDescription;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.result.ITestLifeCycleReceiver;
import com.android.tradefed.result.TestDescription;
@@ -262,6 +263,7 @@
@Test
public void testRun_serialReRunOfTwoFailedToCompleteTests()
throws DeviceNotAvailableException, ConfigurationException {
+ mMockListener = EasyMock.createStrictMock(ITestInvocationListener.class);
final Collection<TestDescription> testsList = new ArrayList<>(1);
final TestDescription test1 = new TestDescription("ClassFoo1", "methodBar1");
final TestDescription test2 = new TestDescription("ClassFoo2", "methodBar2");
@@ -351,6 +353,10 @@
mMockListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mMockListener.testEnded(
EasyMock.eq(test1), EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>()));
+ mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
+ mMockListener.testFailed(EasyMock.eq(test2), EasyMock.<FailureDescription>anyObject());
+ mMockListener.testEnded(
+ EasyMock.eq(test2), EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>()));
mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>()));
// first serial re-run:
mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 0, 1);
diff --git a/tests/src/com/android/tradefed/testtype/InstrumentationTestTest.java b/tests/src/com/android/tradefed/testtype/InstrumentationTestTest.java
index c633824..1c5e61a 100644
--- a/tests/src/com/android/tradefed/testtype/InstrumentationTestTest.java
+++ b/tests/src/com/android/tradefed/testtype/InstrumentationTestTest.java
@@ -465,6 +465,9 @@
inOrder.verify(mMockListener)
.testRunFailed(
FailureDescription.create(RUN_ERROR_MSG, FailureStatus.TEST_FAILURE));
+ inOrder.verify(mMockListener).testStarted(eq(TEST2), anyLong());
+ inOrder.verify(mMockListener).testFailed(eq(TEST2), (FailureDescription) any());
+ inOrder.verify(mMockListener).testEnded(eq(TEST2), anyLong(), eq(EMPTY_STRING_MAP));
inOrder.verify(mMockListener).testRunEnded(1, EMPTY_STRING_MAP);
inOrder.verify(mMockListener).testRunStarted(TEST_PACKAGE_VALUE, 0, 1);