CP: Require 'master' config to be supported in Android CTS
am: 03d217bb53
Change-Id: I6ee82fc14e103e66fef989567c91e8af4b4d1300
diff --git a/android/cts/AndroidTest.xml b/android/cts/AndroidTest.xml
index 75ee8f1..0484be6 100644
--- a/android/cts/AndroidTest.xml
+++ b/android/cts/AndroidTest.xml
@@ -25,6 +25,7 @@
<option name="deqp-surface-type" value="window"/>
<option name="deqp-screen-rotation" value="unspecified"/>
<option name="runtime-hint" value="24m"/>
+ <option name="deqp-config-required" value="true"/>
</test>
<test class="com.drawelements.deqp.runner.DeqpTestRunner">
<option name="deqp-package" value="dEQP-GLES2"/>
@@ -33,6 +34,7 @@
<option name="deqp-surface-type" value="window"/>
<option name="deqp-screen-rotation" value="unspecified"/>
<option name="runtime-hint" value="40m"/>
+ <option name="deqp-config-required" value="true"/>
</test>
<test class="com.drawelements.deqp.runner.DeqpTestRunner">
<option name="deqp-package" value="dEQP-GLES3"/>
@@ -41,6 +43,7 @@
<option name="deqp-surface-type" value="window"/>
<option name="deqp-screen-rotation" value="unspecified"/>
<option name="runtime-hint" value="1h15m"/>
+ <option name="deqp-config-required" value="true"/>
</test>
<test class="com.drawelements.deqp.runner.DeqpTestRunner">
<option name="deqp-package" value="dEQP-GLES3"/>
@@ -97,6 +100,7 @@
<option name="deqp-surface-type" value="window"/>
<option name="deqp-screen-rotation" value="unspecified"/>
<option name="runtime-hint" value="7h30m"/>
+ <option name="deqp-config-required" value="true"/>
</test>
<test class="com.drawelements.deqp.runner.DeqpTestRunner">
<option name="deqp-package" value="dEQP-GLES31"/>
diff --git a/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java
index 280d76d..ca8de1c 100644
--- a/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java
+++ b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java
@@ -115,6 +115,10 @@
description="Surface type ('window', 'pbuffer', 'fbo'). Defaults to 'window'",
importance=Option.Importance.NEVER)
private String mSurfaceType = "window";
+ @Option(name="deqp-config-required",
+ description="Is current config required if API is supported? Defaults to false.",
+ importance=Option.Importance.NEVER)
+ private boolean mConfigRequired = false;
@Option(name = "include-filter",
description="Test include filter. '*' is zero or more letters. '.' has no special meaning.")
private List<String> mIncludeFilters = new ArrayList<>();
@@ -254,19 +258,21 @@
private final String mGlConfig;
private final String mRotation;
private final String mSurfaceType;
+ private final boolean mRequired;
- public BatchRunConfiguration(String glConfig, String rotation, String surfaceType) {
+ public BatchRunConfiguration(String glConfig, String rotation, String surfaceType, boolean required) {
mGlConfig = glConfig;
mRotation = rotation;
mSurfaceType = surfaceType;
+ mRequired = required;
}
/**
* Get string that uniquely identifies this config
*/
public String getId() {
- return String.format("{glformat=%s,rotation=%s,surfacetype=%s}",
- mGlConfig, mRotation, mSurfaceType);
+ return String.format("{glformat=%s,rotation=%s,surfacetype=%s,required=%b}",
+ mGlConfig, mRotation, mSurfaceType, mRequired);
}
/**
@@ -290,6 +296,13 @@
return mSurfaceType;
}
+ /**
+ * Is this configuration mandatory to support, if target API is supported?
+ */
+ public boolean isRequired() {
+ return mRequired;
+ }
+
@Override
public boolean equals(Object other) {
if (other == null) {
@@ -1073,7 +1086,7 @@
}
private static Map<TestIdentifier, Set<BatchRunConfiguration>> generateTestInstances(
- Reader testlist, String configName, String screenRotation, String surfaceType) throws FileNotFoundException {
+ Reader testlist, String configName, String screenRotation, String surfaceType, boolean required) throws FileNotFoundException {
// Note: This is specifically a LinkedHashMap to guarantee that tests are iterated
// in the insertion order.
final Map<TestIdentifier, Set<BatchRunConfiguration>> instances = new LinkedHashMap<>();
@@ -1083,7 +1096,7 @@
while ((testName = testlistReader.readLine()) != null) {
// Test name -> testId -> only one config -> done.
final Set<BatchRunConfiguration> testInstanceSet = new LinkedHashSet<>();
- BatchRunConfiguration config = new BatchRunConfiguration(configName, screenRotation, surfaceType);
+ BatchRunConfiguration config = new BatchRunConfiguration(configName, screenRotation, surfaceType, required);
testInstanceSet.add(config);
TestIdentifier test = pathToIdentifier(testName);
instances.put(test, testInstanceSet);
@@ -1329,7 +1342,11 @@
if (isSupportedRunConfiguration(batch.config)) {
executeTestRunBatch(batch);
} else {
- fakePassTestRunBatch(batch);
+ if (batch.config.isRequired()) {
+ fakeFailTestRunBatch(batch);
+ } else {
+ fakePassTestRunBatch(batch);
+ }
}
}
@@ -1633,13 +1650,24 @@
*/
private void fakePassTestRunBatch(TestBatch batch) {
for (TestIdentifier test : batch.tests) {
- CLog.d("Skipping test '%s' invocation in config '%s'", test.toString(),
+ CLog.d("Marking '%s' invocation in config '%s' as passed without running", test.toString(),
batch.config.getId());
mInstanceListerner.skipTest(test);
}
}
/**
+ * Fail given batch tests without running it
+ */
+ private void fakeFailTestRunBatch(TestBatch batch) {
+ for (TestIdentifier test : batch.tests) {
+ CLog.d("Marking '%s' invocation in config '%s' as failed without running", test.toString(),
+ batch.config.getId());
+ mInstanceListerner.abortTest(test, "Required config not supported");
+ }
+ }
+
+ /**
* Pass all remaining tests without running them
*/
private void fakePassTests(ITestInvocationListener listener) {
@@ -1960,7 +1988,7 @@
}
reader = new FileReader(testlist);
}
- mTestInstances = generateTestInstances(reader, mConfigName, mScreenRotation, mSurfaceType);
+ mTestInstances = generateTestInstances(reader, mConfigName, mScreenRotation, mSurfaceType, mConfigRequired);
mCaselistReader = null;
reader.close();
}
@@ -2083,6 +2111,7 @@
destination.mCaselistFile = source.mCaselistFile;
destination.mScreenRotation = source.mScreenRotation;
destination.mSurfaceType = source.mSurfaceType;
+ destination.mConfigRequired = source.mConfigRequired;
destination.mIncludeFilters = new ArrayList<>(source.mIncludeFilters);
destination.mExcludeFilters = new ArrayList<>(source.mExcludeFilters);
destination.mAbi = source.mAbi;
diff --git a/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java
index 249fcf9..7718d22 100644
--- a/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java
+++ b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java
@@ -407,7 +407,7 @@
if (!pass) {
mockListener.testFailed(testId,
- "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n"
+ "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n"
+ resultCode + ": Detail" + resultCode);
EasyMock.expectLastCall().once();
@@ -931,7 +931,7 @@
EasyMock.expectLastCall().once();
mockListener.testFailed(EasyMock.eq(testIds[i]),
- EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n"
+ EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n"
+ "Abort: Test cannot be executed"));
EasyMock.expectLastCall().once();
diff --git a/scripts/build_android_mustpass.py b/scripts/build_android_mustpass.py
index a20da2c..4139e10 100644
--- a/scripts/build_android_mustpass.py
+++ b/scripts/build_android_mustpass.py
@@ -208,6 +208,7 @@
glconfig = "rgba8888d24s8ms0",
rotation = "unspecified",
surfacetype = "window",
+ required = True,
filters = MASTER_EGL_COMMON_FILTERS,
runtime = "24m"),
])
@@ -223,6 +224,7 @@
glconfig = "rgba8888d24s8ms0",
rotation = "unspecified",
surfacetype = "window",
+ required = True,
filters = MASTER_GLES2_COMMON_FILTERS,
runtime = "40m"),
])
@@ -240,6 +242,7 @@
glconfig = "rgba8888d24s8ms0",
rotation = "unspecified",
surfacetype = "window",
+ required = True,
filters = MASTER_GLES3_COMMON_FILTERS,
runtime = "1h15m"),
# Rotations
@@ -300,6 +303,7 @@
glconfig = "rgba8888d24s8ms0",
rotation = "unspecified",
surfacetype = "window",
+ required = True,
filters = MASTER_GLES31_COMMON_FILTERS,
runtime = "7h30m"),
diff --git a/scripts/mustpass.py b/scripts/mustpass.py
index 0a5f7dd..755faa0 100644
--- a/scripts/mustpass.py
+++ b/scripts/mustpass.py
@@ -42,13 +42,14 @@
self.copyright = copyright
class Configuration:
- def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, runtime = None):
- self.name = name
- self.glconfig = glconfig
- self.rotation = rotation
- self.surfacetype = surfacetype
- self.filters = filters
- self.expectedRuntime = runtime
+ def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None):
+ self.name = name
+ self.glconfig = glconfig
+ self.rotation = rotation
+ self.surfacetype = surfacetype
+ self.required = required
+ self.filters = filters
+ self.expectedRuntime = runtime
class Package:
def __init__ (self, module, configurations):
@@ -387,6 +388,9 @@
if config.expectedRuntime != None:
addOptionElement(testElement, "runtime-hint", config.expectedRuntime)
+ if config.required:
+ addOptionElement(testElement, "deqp-config-required", "true")
+
insertXMLHeaders(mustpass, configElement)
return configElement