Merge "CTS tradefed: ignore known failures." into honeycomb
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/targetsetup/CtsBuildProvider.java b/tools/tradefed-host/src/com/android/cts/tradefed/targetsetup/CtsBuildProvider.java
index 90fb8ba..131c605 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/targetsetup/CtsBuildProvider.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/targetsetup/CtsBuildProvider.java
@@ -50,4 +50,11 @@
public void buildNotTested(IBuildInfo info) {
// ignore
}
+
+ /**
+ * {@inheritDoc}
+ */
+ public void cleanUp(IBuildInfo info) {
+ // ignore
+ }
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ITestCaseRepo.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ITestCaseRepo.java
index 18ea7cf..96546d0 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ITestCaseRepo.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ITestCaseRepo.java
@@ -16,9 +16,6 @@
package com.android.cts.tradefed.testtype;
-import com.android.tradefed.testtype.IRemoteTest;
-
-import java.util.Collection;
/**
* Interface for accessing tests from the CTS repository.
@@ -26,11 +23,11 @@
interface ITestCaseRepo {
/**
- * Gets a list of tests identified by given list of uris
+ * Get a {@link TestPackageDef} given a uri
*
- * @param testUris the string uris
- * @return a {@link Collection} of {@link IRemoteTest}
+ * @param testUri the string uris
+ * @return a {@link TestPackageDef} or <code>null</code> if the uri cannot be found in repo
*/
- public Collection<IRemoteTest> getTests(Collection<String> testUris);
+ public ITestPackageDef getTestPackage(String testUri);
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ITestPackageDef.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ITestPackageDef.java
new file mode 100644
index 0000000..f6febdb
--- /dev/null
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ITestPackageDef.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.cts.tradefed.testtype;
+
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.testtype.IRemoteTest;
+
+import java.io.File;
+
+/**
+ * Container for CTS test info.
+ * <p/>
+ * Knows how to translate this info into a runnable {@link IRemoteTest}.
+ */
+interface ITestPackageDef {
+
+ /**
+ * Get the unique URI of the test package.
+ * @return the {@link String} uri
+ */
+ public String getUri();
+
+ /**
+ * Creates a runnable {@link IRemoteTest} from info stored in this definition.
+ *
+ * @param testCaseDir {@link File} representing directory of test case data
+ * @return a {@link IRemoteTest} with all necessary data populated to run the test or
+ * <code>null</code> if test could not be created
+ */
+ public IRemoteTest createTest(File testCaseDir);
+
+ /**
+ * Determine if given test is defined in this package.
+ *
+ * @param testDef the {@link TestIdentifier}
+ * @return <code>true</code> if test is defined
+ */
+ public boolean isKnownTest(TestIdentifier testDef);
+
+}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/PlanTest.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/PlanTest.java
index 212eb66..8bcf94c 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/PlanTest.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/PlanTest.java
@@ -128,13 +128,14 @@
parser.parse(createXmlStream(ctsPlanFile));
Collection<String> testUris = parser.getTestUris();
ITestCaseRepo testRepo = createTestCaseRepo();
- Collection<IRemoteTest> tests = testRepo.getTests(testUris);
collectDeviceInfo(getDevice(), mTestCaseDir, listeners);
- for (IRemoteTest test : tests) {
- if (test instanceof IDeviceTest) {
- ((IDeviceTest)test).setDevice(getDevice());
+ for (String testUri : testUris) {
+ ITestPackageDef testPackage = testRepo.getTestPackage(testUri);
+ if (testPackage != null) {
+ runTest(listeners, testPackage);
+ } else {
+ Log.w(LOG_TAG, String.format("Could not find test uri %s", testUri));
}
- test.run(listeners);
}
} catch (FileNotFoundException e) {
throw new IllegalArgumentException("failed to find CTS plan file", e);
@@ -144,13 +145,34 @@
}
/**
+ * Runs the test.
+ *
+ * @param listeners
+ * @param testPackage
+ * @throws DeviceNotAvailableException
+ */
+ private void runTest(List<ITestInvocationListener> listeners, ITestPackageDef testPackage)
+ throws DeviceNotAvailableException {
+ IRemoteTest test = testPackage.createTest(mTestCaseDir);
+ if (test != null) {
+ if (test instanceof IDeviceTest) {
+ ((IDeviceTest)test).setDevice(getDevice());
+ }
+ ResultFilter filter = new ResultFilter(listeners, testPackage);
+ test.run(filter);
+ }
+ }
+
+ /**
* Runs the device info collector instrumentation on device, and forwards it to test listeners
* as run metrics.
+ * <p/>
+ * Exposed so unit tests can mock.
*
* @param listeners
* @throws DeviceNotAvailableException
*/
- private void collectDeviceInfo(ITestDevice device, File testApkDir,
+ void collectDeviceInfo(ITestDevice device, File testApkDir,
List<ITestInvocationListener> listeners) throws DeviceNotAvailableException {
DeviceInfoCollector.collectDeviceInfo(device, testApkDir, listeners);
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ResultFilter.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ResultFilter.java
new file mode 100644
index 0000000..d11ab03
--- /dev/null
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ResultFilter.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.cts.tradefed.testtype;
+
+import com.android.ddmlib.Log;
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.result.TestSummary;
+import com.android.tradefed.targetsetup.IBuildInfo;
+
+import java.io.InputStream;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A {@link ITestInvocationListener} that filters test results based on the set of expected tests
+ * in CTS test package xml files.
+ */
+class ResultFilter implements ITestInvocationListener {
+
+ private final List<ITestInvocationListener> mListeners;
+ private final ITestPackageDef mTestPackage;
+
+ /**
+ * Create a {@link ResultFilter}.
+ *
+ * @param listeners the real {@link ITestInvocationListener} to forward results to
+ * @param testPackage the {@link ITestPackageDef} that defines the expected tests
+ */
+ ResultFilter(List<ITestInvocationListener> listeners, ITestPackageDef testPackage) {
+ mListeners = listeners;
+ mTestPackage = testPackage;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void invocationStarted(IBuildInfo buildInfo) {
+ for (ITestInvocationListener listener : mListeners) {
+ listener.invocationStarted(buildInfo);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void invocationFailed(Throwable cause) {
+ for (ITestInvocationListener listener : mListeners) {
+ listener.invocationFailed(cause);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void invocationEnded(long elapsedTime) {
+ for (ITestInvocationListener listener : mListeners) {
+ listener.invocationEnded(elapsedTime);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public TestSummary getSummary() {
+ // should never be called
+ return null;
+ }
+
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testLog(String dataName, LogDataType dataType, InputStream dataStream) {
+ for (ITestInvocationListener listener : mListeners) {
+ listener.testLog(dataName, dataType, dataStream);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testRunStarted(String runName, int testCount) {
+ for (ITestInvocationListener listener : mListeners) {
+ listener.testRunStarted(runName, testCount);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testRunFailed(String errorMessage) {
+ for (ITestInvocationListener listener : mListeners) {
+ listener.testRunFailed(errorMessage);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testRunStopped(long elapsedTime) {
+ for (ITestInvocationListener listener : mListeners) {
+ listener.testRunStopped(elapsedTime);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testRunEnded(long elapsedTime, Map<String, String> runMetrics) {
+ for (ITestInvocationListener listener : mListeners) {
+ listener.testRunEnded(elapsedTime, runMetrics);
+ }
+ // TODO: consider reporting all remaining tests in mTestPackage as failed tests with
+ // notExecuted result
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testStarted(TestIdentifier test) {
+ if (isKnownTest(test)) {
+ for (ITestInvocationListener listener : mListeners) {
+ listener.testStarted(test);
+ }
+ } else {
+ Log.d("ResultFilter", String.format("Skipping reporting unknown test %s", test));
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testFailed(TestFailure status, TestIdentifier test, String trace) {
+ if (isKnownTest(test)) {
+ for (ITestInvocationListener listener : mListeners) {
+ listener.testFailed(status, test, trace);
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testEnded(TestIdentifier test, Map<String, String> testMetrics) {
+ if (isKnownTest(test)) {
+ for (ITestInvocationListener listener : mListeners) {
+ listener.testEnded(test, testMetrics);
+ }
+ }
+ }
+
+ /**
+ * @param test
+ * @return
+ */
+ private boolean isKnownTest(TestIdentifier test) {
+ return mTestPackage.isKnownTest(test);
+ }
+}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestCaseRepo.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestCaseRepo.java
index 8392762..fc20314 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestCaseRepo.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestCaseRepo.java
@@ -16,7 +16,6 @@
package com.android.cts.tradefed.testtype;
import com.android.ddmlib.Log;
-import com.android.tradefed.testtype.IRemoteTest;
import com.android.tradefed.util.xml.AbstractXmlParser.ParseException;
import java.io.BufferedInputStream;
@@ -25,8 +24,6 @@
import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.Hashtable;
import java.util.Map;
@@ -115,21 +112,8 @@
/**
* {@inheritDoc}
*/
- public Collection<IRemoteTest> getTests(Collection<String> testUris) {
- Collection<IRemoteTest> tests = new ArrayList<IRemoteTest>(testUris.size());
- for (String uri : testUris) {
- TestPackageDef def = mTestMap.get(uri);
- if (def != null) {
- IRemoteTest test = def.createTest(mTestCaseDir);
- if (test != null) {
- tests.add(test);
- } else {
- Log.w(LOG_TAG, String.format("Failed to create test from package uri %s", uri));
- }
- } else {
- Log.w(LOG_TAG, String.format("Could not find test with uri %s", uri));
- }
- }
- return tests;
+ @Override
+ public ITestPackageDef getTestPackage(String testUri) {
+ return mTestMap.get(testUri);
}
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
index a75295e..cf67365 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
@@ -29,7 +29,7 @@
* <p/>
* Knows how to translate this info into a runnable {@link IRemoteTest}.
*/
-public class TestPackageDef {
+class TestPackageDef implements ITestPackageDef {
private static final String LOG_TAG = "TestPackageDef";
@@ -44,13 +44,15 @@
private Collection<TestIdentifier> mTests = new ArrayList<TestIdentifier>();
+ /** the cached {@link IRemoteTest} */
+ private IRemoteTest mRemoteTest;
+
void setUri(String uri) {
mUri = uri;
}
/**
- * Get the unique URI of the test package.
- * @return the {@link String} uri
+ * {@inheritDoc}
*/
public String getUri() {
return mUri;
@@ -114,13 +116,20 @@
}
/**
- * Creates a runnable {@link IRemoteTest} from info stored in this definition.
- *
- * @param testCaseDir {@link File} representing directory of test case data
- * @return a {@link IRemoteTest} with all necessary data populated to run the test or
- * <code>null</code> if test could not be created
+ * {@inheritDoc}
*/
public IRemoteTest createTest(File testCaseDir) {
+ if (mRemoteTest == null) {
+ mRemoteTest = doCreateTest(testCaseDir);
+ }
+ return mRemoteTest;
+ }
+
+ /**
+ * @param testCaseDir
+ * @return
+ */
+ private IRemoteTest doCreateTest(File testCaseDir) {
if (mIsHostSideTest) {
Log.d(LOG_TAG, String.format("Creating host test for %s", mName));
JarHostTest hostTest = new JarHostTest();
@@ -157,6 +166,13 @@
}
/**
+ * {@inheritDoc}
+ */
+ public boolean isKnownTest(TestIdentifier testDef) {
+ return mTests.contains(testDef);
+ }
+
+ /**
* Add a {@link TestDef} to the list of tests in this package.
*
* @param testdef