Support Native Test Expectations
Bug 5764723
CTS tests filter out known failures via a text file
containing the "expectations" of tests. This mechanism
was adopted since the libcore tests use the same
system. The recently added native test support didn't
work with the filters though, but now this change adds
that support.
Split the scanning of native tests and XML generation
into separate programs. cts-native-scanner just outputs
test classes and methods it discovers. This is then
piped to cts-xml-generator that takes those tests
and compares them against the test expectations. The
idea is that to support new tests you just need to
add another program that outputs the test classes
and methods. You can then pipe that output to XML
generator and be done.
Having a test expectations is required now, because I
needed to make the XMLs get generated when the test
expectations changed. Before the XMLs were always
regenerated, but now they may not be if nothing has
changed (yay!).
Change-Id: I1feafa3cbcaefdea1b3174ee05026f9c5f58b87e
diff --git a/tools/cts-xml-generator/Android.mk b/tools/cts-xml-generator/Android.mk
new file mode 100644
index 0000000..5842dd8
--- /dev/null
+++ b/tools/cts-xml-generator/Android.mk
@@ -0,0 +1,43 @@
+# Copyright (C) 2011 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LOCAL_PATH := $(call my-dir)
+
+# We use copy-file-to-new-target so that the installed
+# script file's timestamp is at least as new as the
+# .jar file it wraps.
+
+# the hat script
+# ============================================================
+include $(CLEAR_VARS)
+LOCAL_IS_HOST_MODULE := true
+LOCAL_MODULE_CLASS := EXECUTABLES
+LOCAL_MODULE := cts-xml-generator
+LOCAL_MODULE_TAGS := optional
+
+include $(BUILD_SYSTEM)/base_rules.mk
+
+$(LOCAL_BUILT_MODULE): $(HOST_OUT_JAVA_LIBRARIES)/$(LOCAL_MODULE)$(COMMON_JAVA_PACKAGE_SUFFIX)
+$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/etc/$(LOCAL_MODULE) | $(ACP)
+ @echo "Copy: $(PRIVATE_MODULE) ($@)"
+ $(copy-file-to-new-target)
+ $(hide) chmod 755 $@
+
+# the other stuff
+# ============================================================
+subdirs := $(addprefix $(LOCAL_PATH)/,$(addsuffix /Android.mk, \
+ src \
+ ))
+
+include $(subdirs)
diff --git a/tools/cts-xml-generator/etc/cts-xml-generator b/tools/cts-xml-generator/etc/cts-xml-generator
new file mode 100644
index 0000000..5f05af6
--- /dev/null
+++ b/tools/cts-xml-generator/etc/cts-xml-generator
@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# Copyright (C) 2011, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set up prog to be the path of this script, including following symlinks,
+# and set up progdir to be the fully-qualified pathname of its directory.
+prog="$0"
+while [ -h "${prog}" ]; do
+ newProg=`/bin/ls -ld "${prog}"`
+ newProg=`expr "${newProg}" : ".* -> \(.*\)$"`
+ if expr "x${newProg}" : 'x/' >/dev/null; then
+ prog="${newProg}"
+ else
+ progdir=`dirname "${prog}"`
+ prog="${progdir}/${newProg}"
+ fi
+done
+oldwd=`pwd`
+progdir=`dirname "${prog}"`
+cd "${progdir}"
+progdir=`pwd`
+prog="${progdir}"/`basename "${prog}"`
+cd "${oldwd}"
+
+libdir=`dirname $progdir`/framework
+
+javaOpts=""
+while expr "x$1" : 'x-J' >/dev/null; do
+ opt=`expr "$1" : '-J\(.*\)'`
+ javaOpts="${javaOpts} -${opt}"
+ shift
+done
+
+exec java $javaOpts -jar $libdir/cts-xml-generator.jar "$@"
diff --git a/tools/cts-xml-generator/src/Android.mk b/tools/cts-xml-generator/src/Android.mk
new file mode 100644
index 0000000..62f8692
--- /dev/null
+++ b/tools/cts-xml-generator/src/Android.mk
@@ -0,0 +1,30 @@
+# Copyright (C) 2011 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LOCAL_PATH := $(call my-dir)
+
+
+# cts-xml-generator java library
+# ============================================================
+include $(CLEAR_VARS)
+
+LOCAL_SRC_FILES := $(call all-subdir-java-files)
+LOCAL_JAR_MANIFEST := MANIFEST.mf
+
+LOCAL_MODULE := cts-xml-generator
+LOCAL_MODULE_TAGS := optional
+
+LOCAL_STATIC_JAVA_LIBRARIES := vogarexpectlib
+
+include $(BUILD_HOST_JAVA_LIBRARY)
diff --git a/tools/cts-xml-generator/src/MANIFEST.mf b/tools/cts-xml-generator/src/MANIFEST.mf
new file mode 100644
index 0000000..ce4178e
--- /dev/null
+++ b/tools/cts-xml-generator/src/MANIFEST.mf
@@ -0,0 +1,2 @@
+Manifest-Version: 1.0
+Main-Class: com.android.cts.xmlgenerator.CtsXmlGenerator
diff --git a/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/CtsXmlGenerator.java b/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/CtsXmlGenerator.java
new file mode 100644
index 0000000..5d5d8ff
--- /dev/null
+++ b/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/CtsXmlGenerator.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.cts.xmlgenerator;
+
+import vogar.ExpectationStore;
+import vogar.ModeId;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Class that searches a source directory for native gTests and outputs a
+ * test package xml.
+ */
+public class CtsXmlGenerator {
+
+ private static void usage(String[] args) {
+ System.err.println("Arguments: " + Arrays.asList(args));
+ System.err.println("Usage: cts-native-xml-generator -p PACKAGE_NAME -n EXECUTABLE_NAME "
+ + " [-e EXPECTATION_FILE] [-o OUTPUT_FILE]");
+ System.exit(1);
+ }
+
+ public static void main(String[] args) throws Exception {
+ String appPackageName = null;
+ String name = null;
+ String outputPath = null;
+ Set<File> expectationFiles = new HashSet<File>();
+
+ for (int i = 0; i < args.length; i++) {
+ if ("-p".equals(args[i])) {
+ if (i + 1 < args.length) {
+ appPackageName = args[++i];
+ } else {
+ System.err.println("Missing value for test package");
+ usage(args);
+ }
+ } else if ("-n".equals(args[i])) {
+ if (i + 1 < args.length) {
+ name = args[++i];
+ } else {
+ System.err.println("Missing value for executable name");
+ usage(args);
+ }
+ } else if ("-e".equals(args[i])) {
+ if (i + 1 < args.length) {
+ expectationFiles.add(new File(args[++i]));
+ } else {
+ System.err.println("Missing value for expectation store");
+ usage(args);
+ }
+ } else if ("-o".equals(args[i])) {
+ if (i + 1 < args.length) {
+ outputPath = args[++i];
+ } else {
+ System.err.println("Missing value for output file");
+ usage(args);
+ }
+ } else {
+ System.err.println("Unsupported flag: " + args[i]);
+ usage(args);
+ }
+ }
+
+ if (appPackageName == null) {
+ System.out.println("Package name is required");
+ usage(args);
+ } else if (name == null) {
+ System.out.println("Executable name is required");
+ usage(args);
+ }
+
+ ExpectationStore store = ExpectationStore.parse(expectationFiles, ModeId.DEVICE);
+ NativeXmlGenerator generator = new NativeXmlGenerator(store, appPackageName, name, outputPath);
+ generator.writePackageXml();
+ }
+}
diff --git a/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/NativeXmlGenerator.java b/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/NativeXmlGenerator.java
new file mode 100644
index 0000000..746b4db
--- /dev/null
+++ b/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/NativeXmlGenerator.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.cts.xmlgenerator;
+
+import vogar.Expectation;
+import vogar.ExpectationStore;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.util.Scanner;
+
+/**
+ * Generator of TestPackage XML files for native tests.
+ *
+ * It takes in an input of the following form:
+ *
+ * class:TestClass1
+ * method:testMethod1
+ * method:testMethod2
+ * class:TestClass2
+ * method:testMethod1
+ */
+class NativeXmlGenerator {
+
+ /** Test package name like "android.nativemedia" to group the tests. */
+ private final String mAppPackageName;
+
+ /** Name of the native executable. */
+ private final String mName;
+
+ /** Path to output file or null to just dump to standard out. */
+ private final String mOutputPath;
+
+ /** ExpectationStore to filter out known failures. */
+ private final ExpectationStore mExpectations;
+
+ NativeXmlGenerator(ExpectationStore expectations, String appPackageName, String name,
+ String outputPath) {
+ mAppPackageName = appPackageName;
+ mName = name;
+ mOutputPath = outputPath;
+ mExpectations = expectations;
+ }
+
+ public void writePackageXml() throws IOException {
+ OutputStream output = System.out;
+ if (mOutputPath != null) {
+ File outputFile = new File(mOutputPath);
+ File outputDir = outputFile.getParentFile();
+ if (!outputDir.exists()) {
+ outputDir.mkdirs();
+ if (!outputDir.exists()) {
+ System.err.println("Couldn't make output directory: " + outputDir);
+ System.exit(1);
+ }
+ }
+ output = new FileOutputStream(outputFile);
+ }
+
+ PrintWriter writer = null;
+ try {
+ writer = new PrintWriter(output);
+ writer.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+ writeTestPackage(writer);
+ } finally {
+ if (writer != null) {
+ writer.close();
+ }
+ }
+ }
+
+ private void writeTestPackage(PrintWriter writer) {
+ writer.append("<TestPackage appPackageName=\"")
+ .append(mAppPackageName)
+ .append("\" name=\"")
+ .append(mName)
+ .println("\" testType=\"native\" version=\"1.0\">");
+ writeTestSuite(writer);
+ writer.println("</TestPackage>");
+ }
+
+ private void writeTestSuite(PrintWriter writer) {
+ /*
+ * Given "android.foo.bar.baz"...
+ *
+ * <TestSuite name="android">
+ * <TestSuite name="foo">
+ * <TestSuite name="bar">
+ * <TestSuite name="baz">
+ */
+ Scanner scanner = null;
+ try {
+ scanner = new Scanner(mAppPackageName);
+ scanner.useDelimiter("\\.");
+
+ int numLevels = 0;
+ for (; scanner.hasNext(); numLevels++) {
+ String packagePart = scanner.next();
+ writer.append("<TestSuite name=\"").append(packagePart).println("\">");
+ }
+
+ writeTestCases(writer);
+
+ for (; numLevels > 0; numLevels--) {
+ writer.println("</TestSuite>");
+ }
+ } finally {
+ if (scanner != null) {
+ scanner.close();
+ }
+ }
+ }
+
+ private void writeTestCases(PrintWriter writer) {
+ String currentClassName = null;
+ Scanner scanner = new Scanner(System.in);
+ while (scanner.hasNextLine()) {
+ String line = scanner.nextLine();
+ String[] tokens = line.split(":");
+ if (tokens.length > 1) {
+ String type = tokens[0];
+ String value = tokens[1];
+ if ("class".equals(type)) {
+ if (currentClassName != null) {
+ writer.append("</TestCase>");
+ }
+ currentClassName = value;
+ writer.append("<TestCase name=\"").append(value).println("\">");
+ } else if ("method".equals(type)) {
+ String fullClassName = mAppPackageName + "." + currentClassName;
+ if (!isKnownFailure(mExpectations, fullClassName, value)) {
+ writer.append("<Test name=\"").append(value).println("\" />");
+ }
+ }
+ }
+ }
+ if (currentClassName != null) {
+ writer.println("</TestCase>");
+ }
+ }
+
+ public static boolean isKnownFailure(ExpectationStore expectationStore,
+ String className, String methodName) {
+ String testName = String.format("%s#%s", className, methodName);
+ return expectationStore != null && expectationStore.get(testName) != Expectation.SUCCESS;
+ }
+}
+