Use Vogar Expectations to Clean Up Remaining Tests
There were still some core tests that couldn't pass in the overall
CTS run that passed individually. There were also some framework
tests that are also consistently failing.
In the interest of time, these tests will be suppressed. To avoid
a multi-project change, create a expectations file in the CTS
project and use that to filter out some core tests. This approach
actually seems useful, so use to to filter out some of the
framework tests as well! This is probably easier than deleting and
adding annotations!
Change-Id: I93a94c7cf9d388da4fcf0074c6b64688535c25e4
diff --git a/development/ide/eclipse/.classpath b/development/ide/eclipse/.classpath
index 3f11bd1..8cba973 100644
--- a/development/ide/eclipse/.classpath
+++ b/development/ide/eclipse/.classpath
@@ -3,6 +3,7 @@
<classpathentry kind="src" path="cts/apps/CtsVerifier/src"/>
<classpathentry kind="src" path="cts/apps/CtsVerifier/tests/src"/>
<classpathentry kind="src" path="cts/libs/annotation/src"/>
+ <classpathentry kind="src" path="cts/libs/vogar-expect/src"/>
<classpathentry kind="src" path="cts/tests/ApiDemosReferenceTest/src"/>
<classpathentry kind="src" path="cts/tests/ProcessTest/src"/>
<classpathentry kind="src" path="cts/tests/ProcessTest/NoShareUidApp/src"/>
diff --git a/tests/expectations/knownfailures-2.3_r1.txt b/tests/expectations/knownfailures-2.3_r1.txt
new file mode 100644
index 0000000..2be2078
--- /dev/null
+++ b/tests/expectations/knownfailures-2.3_r1.txt
@@ -0,0 +1,34 @@
+[
+/* These tests consistently fail on GRH78. */
+{ name: "android.location.cts.GeocoderTest#testGetFromLocation" },
+{ name: "android.location.cts.GeocoderTest#testGetFromLocationName" },
+{ name: "android.net.cts.ListeningPortsTest#testNoListeningUdp6Ports" },
+{ name: "android.webkit.cts.WebSettingsTest#testSetAppCacheEnabled" },
+{ name: "android.net.cts.SSLCertificateSocketFactoryTest" },
+
+/* These tests pass when executed individually but fail when running CTS as a whole on GRH78. */
+{
+ name: "org.apache.harmony.luni.tests.internal.net.www.protocol.https.HttpsURLConnectionTest#testProxyConnection",
+ bug: 3184701
+},
+{
+ name: "org.apache.harmony.luni.tests.internal.net.www.protocol.https.HttpsURLConnectionTest#testProxyAuthConnection",
+ bug: 3184701
+},
+{
+ name: "org.apache.harmony.luni.tests.internal.net.www.protocol.https.HttpsURLConnectionTest#testConsequentProxyConnection",
+ bug: 3184701
+},
+{
+ name: "org.apache.harmony.luni.tests.internal.net.www.protocol.https.HttpsURLConnectionTest#testProxyAuthConnection_doOutput",
+ bug: 3184701
+},
+{
+ name: "org.apache.harmony.luni.tests.internal.net.www.protocol.https.HttpsURLConnectionTest#testProxyAuthConnectionFailed",
+ bug: 3184701
+},
+{
+ name: "org.apache.harmony.luni.tests.internal.net.www.protocol.https.HttpsURLConnectionTest#testProxyConnection_Not_Found_Response",
+ bug: 3184701
+}
+]
diff --git a/tools/utils/Android.mk b/tools/utils/Android.mk
index 0782116..ddb06c8 100644
--- a/tools/utils/Android.mk
+++ b/tools/utils/Android.mk
@@ -20,7 +20,7 @@
LOCAL_MODULE := descGen
-LOCAL_SRC_FILES := CollectAllTests.java DescriptionGenerator.java
+LOCAL_SRC_FILES := CollectAllTests.java DescriptionGenerator.java VogarUtils.java
LOCAL_CLASSPATH := $(HOST_JDK_TOOLS_JAR) $(LOCAL_PATH)/lib/junit.jar
diff --git a/tools/utils/CollectAllTests.java b/tools/utils/CollectAllTests.java
index cb109e8..c443db2 100644
--- a/tools/utils/CollectAllTests.java
+++ b/tools/utils/CollectAllTests.java
@@ -14,6 +14,15 @@
* limitations under the License.
*/
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import vogar.Expectation;
+import vogar.ExpectationStore;
+import vogar.ModeId;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
@@ -40,15 +49,6 @@
import junit.textui.ResultPrinter;
import junit.textui.TestRunner;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-
-import vogar.ExpectationStore;
-import vogar.Expectation;
-import vogar.ModeId;
-
public class CollectAllTests extends DescriptionGenerator {
static final String ATTRIBUTE_RUNNER = "runner";
@@ -103,12 +103,13 @@
private static String MANIFESTFILE = "";
private static String TESTSUITECLASS = "";
private static String ANDROID_MAKE_FILE = "";
- private static String EXPECTATION_DIR = null;
+ private static String LIBCORE_EXPECTATION_DIR = null;
private static Test TESTSUITE;
static XMLGenerator xmlGenerator;
- private static ExpectationStore vogarExpectationStore;
+ private static ExpectationStore libcoreVogarExpectationStore;
+ private static ExpectationStore ctsVogarExpectationStore;
public static void main(String[] args) {
if (args.length > 2) {
@@ -116,7 +117,7 @@
MANIFESTFILE = args [1];
TESTSUITECLASS = args[2];
if (args.length > 3) {
- EXPECTATION_DIR = args[3];
+ LIBCORE_EXPECTATION_DIR = args[3];
}
if (args.length > 4) {
ANDROID_MAKE_FILE = args[4];
@@ -195,7 +196,8 @@
}
try {
- vogarExpectationStore = provideExpectationStore(EXPECTATION_DIR);
+ libcoreVogarExpectationStore = VogarUtils.provideExpectationStore(LIBCORE_EXPECTATION_DIR);
+ ctsVogarExpectationStore = VogarUtils.provideExpectationStore(CTS_EXPECTATION_DIR);
} catch (IOException e) {
System.err.println("Can't initialize vogar expectation store");
e.printStackTrace(System.err);
@@ -326,15 +328,6 @@
return getAnnotation(testClass, testName, SIDE_EFFECT) != null;
}
- private boolean isVogarKnownFailure(final Class<? extends TestCase> testClass,
- final String testName) {
- if (vogarExpectationStore == null) {
- return false;
- }
- String fullTestName = String.format("%s#%s", testClass.getName(), testName);
- return vogarExpectationStore.get(fullTestName) != Expectation.SUCCESS;
- }
-
private String getAnnotation(final Class<? extends TestCase> testClass,
final String testName, final String annotationName) {
try {
@@ -385,8 +378,11 @@
} else if (hasSideEffects(test.getClass(), testName)) {
System.out.println("ignoring test with side effects: " + test);
return;
- } else if (isVogarKnownFailure(test.getClass(), testName)) {
- System.out.println("ignoring vogar known failure: " + test);
+ } else if (VogarUtils.isVogarKnownFailure(libcoreVogarExpectationStore, test.getClass().getName(), testName)) {
+ System.out.println("ignoring libcore expectation known failure: " + test);
+ return;
+ } else if (VogarUtils.isVogarKnownFailure(ctsVogarExpectationStore, test.getClass().getName(), testName)) {
+ System.out.println("ignoring cts expectation known failure: " + test);
return;
}
@@ -416,26 +412,4 @@
failed.add(test.getClass().getName());
}
}
-
- public static ExpectationStore provideExpectationStore(String dir) throws IOException {
- if (dir == null) {
- return null;
- }
- ExpectationStore result = ExpectationStore.parse(getExpectationFiles(dir), ModeId.DEVICE);
- return result;
- }
-
- private static Set<File> getExpectationFiles(String dir) {
- Set<File> expectSet = new HashSet<File>();
- File[] files = new File(dir).listFiles(new FilenameFilter() {
- // ignore obviously temporary files
- public boolean accept(File dir, String name) {
- return !name.endsWith("~") && !name.startsWith(".");
- }
- });
- if (files != null) {
- expectSet.addAll(Arrays.asList(files));
- }
- return expectSet;
- }
}
diff --git a/tools/utils/DescriptionGenerator.java b/tools/utils/DescriptionGenerator.java
index 2d58543..0731b49 100644
--- a/tools/utils/DescriptionGenerator.java
+++ b/tools/utils/DescriptionGenerator.java
@@ -37,6 +37,8 @@
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
+import vogar.ExpectationStore;
+
import com.sun.javadoc.AnnotationDesc;
import com.sun.javadoc.AnnotationTypeDoc;
import com.sun.javadoc.AnnotationValue;
@@ -67,6 +69,7 @@
static final String BROKEN_TEST = "dalvik.annotation.BrokenTest";
static final String SIDE_EFFECT = "dalvik.annotation.SideEffect";
static final String SUPPRESSED_TEST = "android.test.suitebuilder.annotation.Suppress";
+ static final String CTS_EXPECTATION_DIR = "cts/tests/expectations";
static final String JUNIT_TEST_CASE_CLASS_NAME = "junit.framework.testcase";
static final String TAG_PACKAGE = "TestPackage";
@@ -117,9 +120,17 @@
return true;
}
+ ExpectationStore ctsExpectationStore = null;
+ try {
+ ctsExpectationStore = VogarUtils.provideExpectationStore("./" + CTS_EXPECTATION_DIR);
+ } catch (IOException e) {
+ Log.e("Couldn't load expectation store.", e);
+ return false;
+ }
+
for (ClassDoc clazz : classes) {
if ((!clazz.isAbstract()) && (isValidJUnitTestCase(clazz))) {
- xmlGenerator.addTestClass(new TestClass(clazz));
+ xmlGenerator.addTestClass(new TestClass(clazz, ctsExpectationStore));
}
}
@@ -500,9 +511,9 @@
*
* @param clazz The specified ClassDoc.
*/
- TestClass(ClassDoc clazz) {
+ TestClass(ClassDoc clazz, ExpectationStore expectationStore) {
mName = clazz.toString();
- mCases = getTestMethods(clazz);
+ mCases = getTestMethods(expectationStore, clazz);
}
/**
@@ -511,7 +522,7 @@
* @param clazz The specified ClassDoc.
* @return A collection of TestMethod.
*/
- Collection<TestMethod> getTestMethods(ClassDoc clazz) {
+ Collection<TestMethod> getTestMethods(ExpectationStore expectationStore, ClassDoc clazz) {
Collection<MethodDoc> methods = getAllMethods(clazz);
ArrayList<TestMethod> cases = new ArrayList<TestMethod>();
@@ -541,6 +552,10 @@
}
}
+ if (VogarUtils.isVogarKnownFailure(expectationStore, clazz.toString(), name)) {
+ isBroken = true;
+ }
+
if (name.startsWith("test")) {
cases.add(new TestMethod(name, method.commentText(), controller, knownFailure,
isBroken, isSuppressed));
diff --git a/tools/utils/VogarUtils.java b/tools/utils/VogarUtils.java
new file mode 100644
index 0000000..06b48c6
--- /dev/null
+++ b/tools/utils/VogarUtils.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import vogar.Expectation;
+import vogar.ExpectationStore;
+import vogar.ModeId;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+public class VogarUtils {
+
+ public static boolean isVogarKnownFailure(ExpectationStore expectationStore,
+ final String testClassName,
+ final String testMethodName) {
+ String fullTestName = String.format("%s#%s", testClassName, testMethodName);
+ return expectationStore != null
+ && expectationStore.get(fullTestName) != Expectation.SUCCESS;
+ }
+
+ public static ExpectationStore provideExpectationStore(String dir) throws IOException {
+ if (dir == null) {
+ return null;
+ }
+ ExpectationStore result = ExpectationStore.parse(getExpectationFiles(dir), ModeId.DEVICE);
+ return result;
+ }
+
+ private static Set<File> getExpectationFiles(String dir) {
+ Set<File> expectSet = new HashSet<File>();
+ File[] files = new File(dir).listFiles(new FilenameFilter() {
+ // ignore obviously temporary files
+ public boolean accept(File dir, String name) {
+ return !name.endsWith("~") && !name.startsWith(".");
+ }
+ });
+ if (files != null) {
+ expectSet.addAll(Arrays.asList(files));
+ }
+ return expectSet;
+ }
+}