Support test instancing in DeqpTestRunner.
* Add support for test instances with different render target
configurations. A test case will be reported as a failure if any of
its instances reports a failure or terminates unexpectedly.
* Check support for orientation and pixel format before attempting to
run test instances. Report unsupported instances as passes.
* Update dEQP test package XML abi attribute injector regex to support
subelements within Test-elements.
Bug: 18982383
Change-Id: I67c65caabef8e1c0ecc9dccfb409a3a19bcff4e3
diff --git a/build/test_deqp_package.mk b/build/test_deqp_package.mk
index b07876d..58df98f 100644
--- a/build/test_deqp_package.mk
+++ b/build/test_deqp_package.mk
@@ -39,6 +39,6 @@
| grep --only-matching -e " abis=\"[^\"]*\""))
# Patch xml caselist with supported abi
- $(hide) $(SED_EXTENDED) -e 's:^<Test (.*)/>$$:<Test \1 $(supported_abi_attr) />:' \
+ $(hide) $(SED_EXTENDED) -e 's:^<Test ((.[^/]|[^/])*)(/?)>$$:<Test \1 $(supported_abi_attr) \3>:' \
< $(MUSTPASS_XML_FILE) \
> $@
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/DeqpTestRunner.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/DeqpTestRunner.java
index 8eb1621..6d8d807 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/DeqpTestRunner.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/DeqpTestRunner.java
@@ -22,8 +22,14 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* Test runner for dEQP tests
@@ -35,29 +41,35 @@
private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
private static final String INCOMPLETE_LOG_MESSAGE = "Crash: Incomplete test log";
+ private static final String DEVICE_LOST_MESSAGE = "Crash: Device lost";
+ private static final String SKIPPED_INSTANCE_LOG_MESSAGE = "Configuration skipped";
+ private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt";
+ private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa";
+ public static final String FEATURE_LANDSCAPE = "android.hardware.screen.landscape";
+ public static final String FEATURE_PORTRAIT = "android.hardware.screen.portrait";
- private final int TESTCASE_BATCH_LIMIT = 1000;
-
- private boolean mLogData;
-
- private ITestDevice mDevice;
+ private static final int TESTCASE_BATCH_LIMIT = 1000;
+ private static final BatchRunConfiguration DEFAULT_CONFIG =
+ new BatchRunConfiguration("rgba8888d24s8", "unspecified", "window");
private final String mPackageName;
private final String mName;
- private Collection<TestIdentifier> mTests;
+ private final Collection<TestIdentifier> mRemainingTests;
+ private final Map<TestIdentifier, Set<BatchRunConfiguration>> mTestInstances;
+ private final TestInstanceResultListener mInstanceListerner;
private IAbi mAbi;
private CtsBuildHelper mCtsBuild;
+ private boolean mLogData;
+ private ITestDevice mDevice;
+ private Set<String> mDeviceFeatures;
- private TestIdentifier mCurrentTestId;
- private boolean mGotTestResult;
- private String mCurrentTestLog;
-
- private ITestInvocationListener mListener;
-
- public DeqpTestRunner(String packageName, String name, Collection<TestIdentifier> tests) {
+ public DeqpTestRunner(String packageName, String name, Collection<TestIdentifier> tests,
+ Map<TestIdentifier, List<Map<String,String>>> testInstances) {
mPackageName = packageName;
mName = name;
- mTests = tests;
+ mRemainingTests = new LinkedList<>(tests); // avoid modifying arguments
+ mTestInstances = parseTestInstances(tests, testInstances);
+ mInstanceListerner = new TestInstanceResultListener();
mLogData = false;
}
@@ -95,18 +107,390 @@
}
/**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ mDevice = device;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ITestDevice getDevice() {
+ return mDevice;
+ }
+
+ private static final class CapabilityQueryFailureException extends Exception {
+ };
+
+ /**
+ * Test configuration of dEPQ test instance execution.
+ * Exposed for unit testing
+ */
+ public static final class BatchRunConfiguration {
+ public static final String ROTATION_UNSPECIFIED = "unspecified";
+ public static final String ROTATION_PORTRAIT = "0";
+ public static final String ROTATION_LANDSCAPE = "90";
+ public static final String ROTATION_REVERSE_PORTRAIT = "180";
+ public static final String ROTATION_REVERSE_LANDSCAPE = "270";
+
+ private final String mGlConfig;
+ private final String mRotation;
+ private final String mSurfaceType;
+
+ public BatchRunConfiguration(String glConfig, String rotation, String surfaceType) {
+ mGlConfig = glConfig;
+ mRotation = rotation;
+ mSurfaceType = surfaceType;
+ }
+
+ /**
+ * Get string that uniquely identifies this config
+ */
+ public String getId() {
+ return String.format("{glformat=%s,rotation=%s,surfacetype=%s}",
+ mGlConfig, mRotation, mSurfaceType);
+ }
+
+ /**
+ * Get the GL config used in this configuration.
+ */
+ public String getGlConfig() {
+ return mGlConfig;
+ }
+
+ /**
+ * Get the screen rotation used in this configuration.
+ */
+ public String getRotation() {
+ return mRotation;
+ }
+
+ /**
+ * Get the surface type used in this configuration.
+ */
+ public String getSurfaceType() {
+ return mSurfaceType;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ } else if (!(other instanceof BatchRunConfiguration)) {
+ return false;
+ } else {
+ return getId().equals(((BatchRunConfiguration)other).getId());
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getId().hashCode();
+ }
+ }
+
+ /**
+ * dEQP test instance listerer and invocation result forwarded
+ */
+ private class TestInstanceResultListener {
+ private ITestInvocationListener mSink;
+ private BatchRunConfiguration mRunConfig;
+
+ private TestIdentifier mCurrentTestId;
+ private boolean mGotTestResult;
+ private String mCurrentTestLog;
+
+ private class PendingResult
+ {
+ boolean allInstancesPassed;
+ Map<BatchRunConfiguration, String> testLogs;
+ Map<BatchRunConfiguration, String> errorMessages;
+ Set<BatchRunConfiguration> remainingConfigs;
+ };
+ private final Map<TestIdentifier, PendingResult> mPendingResults = new HashMap<>();
+
+ public void setSink(ITestInvocationListener sink) {
+ mSink = sink;
+ }
+
+ public void setCurrentConfig(BatchRunConfiguration runConfig) {
+ mRunConfig = runConfig;
+ }
+
+ /**
+ * Forward result to sink
+ */
+ private void forwardFinalizedPendingResult() {
+ if (mRemainingTests.contains(mCurrentTestId)) {
+ final PendingResult result = mPendingResults.get(mCurrentTestId);
+
+ mRemainingTests.remove(mCurrentTestId);
+ mSink.testStarted(mCurrentTestId);
+
+ // Test Log
+ if (mLogData) {
+ for (Map.Entry<BatchRunConfiguration, String> entry :
+ result.testLogs.entrySet()) {
+ final ByteArrayInputStreamSource source
+ = new ByteArrayInputStreamSource(entry.getValue().getBytes());
+
+ mSink.testLog(mCurrentTestId.getClassName() + "."
+ + mCurrentTestId.getTestName() + "@" + entry.getKey().getId(),
+ LogDataType.XML, source);
+
+ source.cancel();
+ }
+ }
+
+ // Error message
+ if (!result.allInstancesPassed) {
+ final StringBuilder errorLog = new StringBuilder();
+
+ for (Map.Entry<BatchRunConfiguration, String> entry :
+ result.errorMessages.entrySet()) {
+ if (errorLog.length() > 0) {
+ errorLog.append('\n');
+ }
+ errorLog.append(String.format("=== with config %s ===\n",
+ entry.getKey().getId()));
+ errorLog.append(entry.getValue());
+ }
+
+ mSink.testFailed(mCurrentTestId, errorLog.toString());
+ }
+
+ // Clear all that won't be used again. The memory usage of these might
+ // add up to quite large numbers
+ result.testLogs = null;
+ result.errorMessages = null;
+ final Map<String, String> emptyMap = Collections.emptyMap();
+ mSink.testEnded(mCurrentTestId, emptyMap);
+ }
+ }
+
+ /**
+ * Declare existence of a test and instances
+ */
+ public void setTestInstances(TestIdentifier testId, Set<BatchRunConfiguration> configs) {
+ // Test instances cannot change at runtime, ignore if we have already set this
+ if (!mPendingResults.containsKey(testId)) {
+ final PendingResult pendingResult = new PendingResult();
+ pendingResult.allInstancesPassed = true;
+ pendingResult.testLogs = new LinkedHashMap<>();
+ pendingResult.errorMessages = new LinkedHashMap<>();
+ pendingResult.remainingConfigs = new HashSet<>(configs); // avoid mutating argument
+ mPendingResults.put(testId, pendingResult);
+ }
+ }
+
+ /**
+ * Query if test instance has not yet been executed
+ */
+ public boolean isPendingTestInstance(TestIdentifier testId,
+ BatchRunConfiguration config) {
+ final PendingResult result = mPendingResults.get(testId);
+ return result.remainingConfigs.contains(config);
+ }
+
+ /**
+ * Fake execution of an instance with current config
+ */
+ public void skipTest(TestIdentifier testId) {
+ final PendingResult result = mPendingResults.get(testId);
+
+ result.errorMessages.put(mRunConfig, SKIPPED_INSTANCE_LOG_MESSAGE);
+ result.remainingConfigs.remove(mRunConfig);
+
+ if (result.remainingConfigs.isEmpty()) {
+ // fake as if we actually run the test
+ mCurrentTestId = testId;
+ forwardFinalizedPendingResult();
+ mCurrentTestId = null;
+ }
+ }
+
+ /**
+ * Handles beginning of dEQP session.
+ */
+ private void handleBeginSession(Map<String, String> values) {
+ // ignore
+ }
+
+ /**
+ * Handles end of dEQP session.
+ */
+ private void handleEndSession(Map<String, String> values) {
+ // ignore
+ }
+
+ /**
+ * Handles beginning of dEQP testcase.
+ */
+ private void handleBeginTestCase(Map<String, String> values) {
+ mCurrentTestId = pathToIdentifier(values.get("dEQP-BeginTestCase-TestCasePath"));
+ mCurrentTestLog = "";
+ mGotTestResult = false;
+
+ // mark instance as started
+ mPendingResults.get(mCurrentTestId).remainingConfigs.remove(mRunConfig);
+ }
+
+ /**
+ * Handles end of dEQP testcase.
+ */
+ private void handleEndTestCase(Map<String, String> values) {
+ final PendingResult result = mPendingResults.get(mCurrentTestId);
+ if (!mGotTestResult) {
+ result.allInstancesPassed = false;
+ result.errorMessages.put(mRunConfig, INCOMPLETE_LOG_MESSAGE);
+ }
+
+ if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
+ result.testLogs.put(mRunConfig, mCurrentTestLog);
+ }
+
+ // Pending result finished, report result
+ if (result.remainingConfigs.isEmpty()) {
+ forwardFinalizedPendingResult();
+ }
+ mCurrentTestId = null;
+ }
+
+ /**
+ * Handles dEQP testcase result.
+ */
+ private void handleTestCaseResult(Map<String, String> values) {
+ String code = values.get("dEQP-TestCaseResult-Code");
+ String details = values.get("dEQP-TestCaseResult-Details");
+
+ if (code.compareTo("Pass") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("NotSupported") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("QualityWarning") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("CompatibilityWarning") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("Fail") == 0 || code.compareTo("ResourceError") == 0
+ || code.compareTo("InternalError") == 0 || code.compareTo("Crash") == 0
+ || code.compareTo("Timeout") == 0) {
+ mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
+ mPendingResults.get(mCurrentTestId)
+ .errorMessages.put(mRunConfig, code + ": " + details);
+ mGotTestResult = true;
+ } else {
+ String codeError = "Unknown result code: " + code;
+ mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
+ mPendingResults.get(mCurrentTestId)
+ .errorMessages.put(mRunConfig, codeError + ": " + details);
+ mGotTestResult = true;
+ }
+ }
+
+ /**
+ * Handles terminated dEQP testcase.
+ */
+ private void handleTestCaseTerminate(Map<String, String> values) {
+ final PendingResult result = mPendingResults.get(mCurrentTestId);
+
+ String reason = values.get("dEQP-TerminateTestCase-Reason");
+ mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
+ mPendingResults.get(mCurrentTestId)
+ .errorMessages.put(mRunConfig, "Terminated: " + reason);
+
+ // Pending result finished, report result
+ if (result.remainingConfigs.isEmpty()) {
+ forwardFinalizedPendingResult();
+ }
+
+ mCurrentTestId = null;
+ mGotTestResult = true;
+ }
+
+ /**
+ * Handles dEQP testlog data.
+ */
+ private void handleTestLogData(Map<String, String> values) {
+ mCurrentTestLog = mCurrentTestLog + values.get("dEQP-TestLogData-Log");
+ }
+
+ /**
+ * Handles new instrumentation status message.
+ */
+ public void handleStatus(Map<String, String> values) {
+ String eventType = values.get("dEQP-EventType");
+
+ if (eventType == null) {
+ return;
+ }
+
+ if (eventType.compareTo("BeginSession") == 0) {
+ handleBeginSession(values);
+ } else if (eventType.compareTo("EndSession") == 0) {
+ handleEndSession(values);
+ } else if (eventType.compareTo("BeginTestCase") == 0) {
+ handleBeginTestCase(values);
+ } else if (eventType.compareTo("EndTestCase") == 0) {
+ handleEndTestCase(values);
+ } else if (eventType.compareTo("TestCaseResult") == 0) {
+ handleTestCaseResult(values);
+ } else if (eventType.compareTo("TerminateTestCase") == 0) {
+ handleTestCaseTerminate(values);
+ } else if (eventType.compareTo("TestLogData") == 0) {
+ handleTestLogData(values);
+ }
+ }
+
+ /**
+ * Signal listener that batch ended to flush incomplete results.
+ */
+ public void endBatch() {
+ // end open test if when stream ends
+ if (mCurrentTestId != null) {
+ final Map<String, String> emptyMap = Collections.emptyMap();
+ handleEndTestCase(emptyMap);
+ }
+ }
+
+ /**
+ * Signal listener that device just died.
+ */
+ public void onDeviceLost() {
+ if (mCurrentTestId != null) {
+ final PendingResult result = mPendingResults.get(mCurrentTestId);
+
+ // kill current test
+ result.allInstancesPassed = false;
+ result.errorMessages.put(mRunConfig, DEVICE_LOST_MESSAGE);
+
+ if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
+ result.testLogs.put(mRunConfig, mCurrentTestLog);
+ }
+
+ // finish all pending instances
+ result.remainingConfigs.clear();
+ forwardFinalizedPendingResult();
+ mCurrentTestId = null;
+ }
+ }
+ }
+
+ /**
* dEQP instrumentation parser
*/
- class InstrumentationParser extends MultiLineReceiver {
- private DeqpTestRunner mDeqpTests;
+ private static class InstrumentationParser extends MultiLineReceiver {
+ private TestInstanceResultListener mListener;
private Map<String, String> mValues;
private String mCurrentName;
private String mCurrentValue;
- public InstrumentationParser(DeqpTestRunner tests) {
- mDeqpTests = tests;
+ public InstrumentationParser(TestInstanceResultListener listener) {
+ mListener = listener;
}
/**
@@ -125,7 +509,7 @@
mCurrentValue = null;
}
- mDeqpTests.handleStatus(mValues);
+ mListener.handleStatus(mValues);
mValues = null;
} else if (line.startsWith("INSTRUMENTATION_STATUS: dEQP-")) {
if (mCurrentName != null) {
@@ -161,7 +545,7 @@
}
if (mValues != null) {
- mDeqpTests.handleStatus(mValues);
+ mListener.handleStatus(mValues);
mValues = null;
}
}
@@ -176,9 +560,112 @@
}
/**
+ * dEQP platfom query instrumentation parser
+ */
+ private static class PlatformQueryInstrumentationParser extends MultiLineReceiver {
+ private Map<String,String> mResultMap = new LinkedHashMap<>();
+ private int mResultCode;
+ private boolean mGotExitValue = false;
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void processNewLines(String[] lines) {
+ for (String line : lines) {
+ if (line.startsWith("INSTRUMENTATION_RESULT: ")) {
+ final String parts[] = line.substring(24).split("=",2);
+ if (parts.length == 2) {
+ mResultMap.put(parts[0], parts[1]);
+ } else {
+ CLog.w("Instrumentation status format unexpected");
+ }
+ } else if (line.startsWith("INSTRUMENTATION_CODE: ")) {
+ try {
+ mResultCode = Integer.parseInt(line.substring(22));
+ mGotExitValue = true;
+ } catch (NumberFormatException ex) {
+ CLog.w("Instrumentation code format unexpected");
+ }
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+
+ public boolean wasSuccessful() {
+ return mGotExitValue;
+ }
+
+ public int getResultCode() {
+ return mResultCode;
+ }
+
+ public Map<String,String> getResultMap() {
+ return mResultMap;
+ }
+ }
+
+ /**
+ * Parse map of instance arguments to map of BatchRunConfigurations
+ */
+ private static Map<TestIdentifier, Set<BatchRunConfiguration>> parseTestInstances(
+ Collection<TestIdentifier> tests,
+ Map<TestIdentifier, List<Map<String,String>>> testInstances) {
+ final Map<TestIdentifier, Set<BatchRunConfiguration>> instances = new HashMap<>();
+ for (final TestIdentifier test : tests) {
+ final Set<BatchRunConfiguration> testInstanceSet = new LinkedHashSet<>();
+ if (testInstances.get(test).isEmpty()) {
+ // no instances defined, use default
+ testInstanceSet.add(DEFAULT_CONFIG);
+ } else {
+ for (Map<String, String> instanceArgs : testInstances.get(test)) {
+ testInstanceSet.add(parseRunConfig(instanceArgs));
+ }
+ }
+ instances.put(test, testInstanceSet);
+ }
+ return instances;
+ }
+
+ private static BatchRunConfiguration parseRunConfig(Map<String,String> instanceArguments) {
+ final String glConfig;
+ final String rotation;
+ final String surfaceType;
+
+ if (instanceArguments.containsKey("glconfig")) {
+ glConfig = instanceArguments.get("glconfig");
+ } else {
+ glConfig = DEFAULT_CONFIG.getGlConfig();
+ }
+ if (instanceArguments.containsKey("rotation")) {
+ rotation = instanceArguments.get("rotation");
+ } else {
+ rotation = DEFAULT_CONFIG.getRotation();
+ }
+ if (instanceArguments.containsKey("surfaceType")) {
+ surfaceType = instanceArguments.get("surfaceType");
+ } else {
+ surfaceType = DEFAULT_CONFIG.getSurfaceType();
+ }
+
+ return new BatchRunConfiguration(glConfig, rotation, surfaceType);
+ }
+
+ private Set<BatchRunConfiguration> getTestRunConfigs (TestIdentifier testId) {
+ return mTestInstances.get(testId);
+ }
+
+ /**
* Converts dEQP testcase path to TestIdentifier.
*/
- private TestIdentifier pathToIdentifier(String testPath) {
+ private static TestIdentifier pathToIdentifier(String testPath) {
String[] components = testPath.split("\\.");
String name = components[components.length - 1];
String className = null;
@@ -194,150 +681,14 @@
return new TestIdentifier(className, name);
}
- /**
- * Handles beginning of dEQP session.
- */
- private void handleBeginSession(Map<String, String> values) {
- String id = AbiUtils.createId(mAbi.getName(), mPackageName);
- mListener.testRunStarted(id, mTests.size());
- }
-
- /**
- * Handles end of dEQP session.
- */
- private void handleEndSession(Map<String, String> values) {
- Map <String, String> emptyMap = Collections.emptyMap();
- mListener.testRunEnded(0, emptyMap);
- }
-
- /**
- * Handles beginning of dEQP testcase.
- */
- private void handleBeginTestCase(Map<String, String> values) {
- mCurrentTestId = pathToIdentifier(values.get("dEQP-BeginTestCase-TestCasePath"));
- mCurrentTestLog = "";
- mGotTestResult = false;
-
- mListener.testStarted(mCurrentTestId);
- mTests.remove(mCurrentTestId);
- }
-
- /**
- * Handles end of dEQP testcase.
- */
- private void handleEndTestCase(Map<String, String> values) {
- Map <String, String> emptyMap = Collections.emptyMap();
-
- if (!mGotTestResult) {
- mListener.testFailed(mCurrentTestId,
- INCOMPLETE_LOG_MESSAGE);
- }
-
- if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
- ByteArrayInputStreamSource source
- = new ByteArrayInputStreamSource(mCurrentTestLog.getBytes());
-
- mListener.testLog(mCurrentTestId.getClassName() + "."
- + mCurrentTestId.getTestName(), LogDataType.XML, source);
-
- source.cancel();
- }
-
- mListener.testEnded(mCurrentTestId, emptyMap);
- mCurrentTestId = null;
- }
-
- /**
- * Handles dEQP testcase result.
- */
- private void handleTestCaseResult(Map<String, String> values) {
- String code = values.get("dEQP-TestCaseResult-Code");
- String details = values.get("dEQP-TestCaseResult-Details");
-
- if (code.compareTo("Pass") == 0) {
- mGotTestResult = true;
- } else if (code.compareTo("NotSupported") == 0) {
- mGotTestResult = true;
- } else if (code.compareTo("QualityWarning") == 0) {
- mGotTestResult = true;
- } else if (code.compareTo("CompatibilityWarning") == 0) {
- mGotTestResult = true;
- } else if (code.compareTo("Fail") == 0 || code.compareTo("ResourceError") == 0
- || code.compareTo("InternalError") == 0 || code.compareTo("Crash") == 0
- || code.compareTo("Timeout") == 0) {
- mListener.testFailed(mCurrentTestId,
- code + ": " + details);
- mGotTestResult = true;
- } else {
- mListener.testFailed(mCurrentTestId,
- "Unknown result code: " + code + ": " + details);
- mGotTestResult = true;
- }
- }
-
- /**
- * Handles terminated dEQP testcase.
- */
- private void handleTestCaseTerminate(Map<String, String> values) {
- Map <String, String> emptyMap = Collections.emptyMap();
-
- String reason = values.get("dEQP-TerminateTestCase-Reason");
- mListener.testFailed(mCurrentTestId,
- "Terminated: " + reason);
- mListener.testEnded(mCurrentTestId, emptyMap);
-
- if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
- ByteArrayInputStreamSource source
- = new ByteArrayInputStreamSource(mCurrentTestLog.getBytes());
-
- mListener.testLog(mCurrentTestId.getClassName() + "."
- + mCurrentTestId.getTestName(), LogDataType.XML, source);
-
- source.cancel();
- }
-
- mCurrentTestId = null;
- mGotTestResult = true;
- }
-
- /**
- * Handles dEQP testlog data.
- */
- private void handleTestLogData(Map<String, String> values) {
- mCurrentTestLog = mCurrentTestLog + values.get("dEQP-TestLogData-Log");
- }
-
- /**
- * Handles new instrumentation status message.
- */
- public void handleStatus(Map<String, String> values) {
- String eventType = values.get("dEQP-EventType");
-
- if (eventType == null) {
- return;
- }
-
- if (eventType.compareTo("BeginSession") == 0) {
- handleBeginSession(values);
- } else if (eventType.compareTo("EndSession") == 0) {
- handleEndSession(values);
- } else if (eventType.compareTo("BeginTestCase") == 0) {
- handleBeginTestCase(values);
- } else if (eventType.compareTo("EndTestCase") == 0) {
- handleEndTestCase(values);
- } else if (eventType.compareTo("TestCaseResult") == 0) {
- handleTestCaseResult(values);
- } else if (eventType.compareTo("TerminateTestCase") == 0) {
- handleTestCaseTerminate(values);
- } else if (eventType.compareTo("TestLogData") == 0) {
- handleTestLogData(values);
- }
+ private String getId() {
+ return AbiUtils.createId(mAbi.getName(), mPackageName);
}
/**
* Generates tescase trie from dEQP testcase paths. Used to define which testcases to execute.
*/
- private String generateTestCaseTrieFromPaths(Collection<String> tests) {
+ private static String generateTestCaseTrieFromPaths(Collection<String> tests) {
String result = "{";
boolean first = true;
@@ -390,15 +741,11 @@
/**
* Generates testcase trie from TestIdentifiers.
*/
- private String generateTestCaseTrie(Collection<TestIdentifier> tests) {
+ private static String generateTestCaseTrie(Collection<TestIdentifier> tests) {
ArrayList<String> testPaths = new ArrayList<String>();
for (TestIdentifier test : tests) {
testPaths.add(test.getClassName() + "." + test.getTestName());
-
- // Limit number of testcases for each run
- if (testPaths.size() > TESTCASE_BATCH_LIMIT)
- break;
}
return generateTestCaseTrieFromPaths(testPaths);
@@ -407,34 +754,178 @@
/**
* Executes tests on the device.
*/
- private void executeTests(ITestInvocationListener listener) throws DeviceNotAvailableException {
- InstrumentationParser parser = new InstrumentationParser(this);
- String caseListFileName = "/sdcard/dEQP-TestCaseList.txt";
- String logFileName = "/sdcard/TestLog.qpa";
- String testCases = generateTestCaseTrie(mTests);
+ private void runTests() throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ while (!mRemainingTests.isEmpty()) {
+ // select tests for the batch
+ final ArrayList<TestIdentifier> batchTests = new ArrayList<>(TESTCASE_BATCH_LIMIT);
+ for (TestIdentifier test : mRemainingTests) {
+ batchTests.add(test);
+ if (batchTests.size() >= TESTCASE_BATCH_LIMIT) {
+ break;
+ }
+ }
- mDevice.executeShellCommand("rm " + caseListFileName);
- mDevice.executeShellCommand("rm " + logFileName);
- mDevice.pushString(testCases + "\n", caseListFileName);
+ // find union of all run configurations
+ final Set<BatchRunConfiguration> allConfigs = new LinkedHashSet<>();
+ for (TestIdentifier test : batchTests) {
+ allConfigs.addAll(getTestRunConfigs(test));
+ }
- String instrumentationName =
+ // prepare instance listener
+ for (TestIdentifier test : batchTests) {
+ mInstanceListerner.setTestInstances(test, getTestRunConfigs(test));
+ }
+
+ // run batch for all configurations
+ for (BatchRunConfiguration runConfig : allConfigs) {
+ final ArrayList<TestIdentifier> relevantTests =
+ new ArrayList<>(TESTCASE_BATCH_LIMIT);
+
+ // run only for declared run configs and only if test has not already
+ // been attempted to run
+ for (TestIdentifier test : batchTests) {
+ if (mInstanceListerner.isPendingTestInstance(test, runConfig)) {
+ relevantTests.add(test);
+ }
+ }
+
+ if (!relevantTests.isEmpty()) {
+ runTestRunBatch(relevantTests, runConfig);
+ }
+ }
+ }
+ }
+
+ private void runTestRunBatch(Collection<TestIdentifier> tests, BatchRunConfiguration runConfig)
+ throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ boolean isSupportedConfig = true;
+
+ // orientation support
+ if (!BatchRunConfiguration.ROTATION_UNSPECIFIED.equals(runConfig.getRotation())) {
+ final Set<String> features = getDeviceFeatures(mDevice);
+
+ if (isPortraitClassRotation(runConfig.getRotation()) &&
+ !features.contains(FEATURE_PORTRAIT)) {
+ isSupportedConfig = false;
+ }
+ if (isLandscapeClassRotation(runConfig.getRotation()) &&
+ !features.contains(FEATURE_LANDSCAPE)) {
+ isSupportedConfig = false;
+ }
+ }
+
+ // renderability support for OpenGL ES tests
+ if (isSupportedConfig && isOpenGlEsPackage()) {
+ isSupportedConfig = isSupportedGlesRenderConfig(runConfig);
+ }
+
+ mInstanceListerner.setCurrentConfig(runConfig);
+
+ // execute only if config is executable, else fake results
+ if (isSupportedConfig) {
+ executeTestRunBatch(tests, runConfig);
+ } else {
+ fakePassTestRunBatch(tests, runConfig);
+ }
+ }
+
+ private void executeTestRunBatch(Collection<TestIdentifier> tests,
+ BatchRunConfiguration runConfig) throws DeviceNotAvailableException {
+ final String testCases = generateTestCaseTrie(tests);
+
+ mDevice.executeShellCommand("rm " + CASE_LIST_FILE_NAME);
+ mDevice.executeShellCommand("rm " + LOG_FILE_NAME);
+ mDevice.pushString(testCases + "\n", CASE_LIST_FILE_NAME);
+
+ final String instrumentationName =
"com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
- String command = String.format(
- "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \""
- + "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8\""
- + " -e deqpLogData \"%s\" %s",
- AbiUtils.createAbiFlag(mAbi.getName()), logFileName, caseListFileName, mLogData,
- instrumentationName);
+ final StringBuilder deqpCmdLine = new StringBuilder();
+ deqpCmdLine.append("--deqp-caselist-file=");
+ deqpCmdLine.append(CASE_LIST_FILE_NAME);
+ deqpCmdLine.append(" ");
+ deqpCmdLine.append(getRunConfigDisplayCmdLine(runConfig));
- mDevice.executeShellCommand(command, parser);
- parser.flush();
+ final String command = String.format(
+ "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\""
+ + " -e deqpLogData \"%s\" %s",
+ AbiUtils.createAbiFlag(mAbi.getName()), LOG_FILE_NAME, deqpCmdLine.toString(),
+ mLogData, instrumentationName);
+
+ try {
+ final InstrumentationParser parser = new InstrumentationParser(mInstanceListerner);
+ mDevice.executeShellCommand(command, parser);
+ parser.flush();
+ } catch (DeviceNotAvailableException ex) {
+ // Device lost. We must signal the tradedef by rethrowing this execption. However,
+ // there is a possiblity that the device loss was caused by the currently run test
+ // instance. Since CtsTest is unaware of tests with only some instances executed,
+ // continuing the session after device has recovered will create a new DeqpTestRunner
+ // with current test in its run queue and this will cause the re-execution of this same
+ // instance. If the instance reliably can kill the device, the CTS cannot recover.
+ //
+ // Prevent this by terminating ALL instances of a tests if any of them causes a device
+ // loss.
+ mInstanceListerner.onDeviceLost();
+ throw ex;
+ } finally {
+ mInstanceListerner.endBatch();
+ }
+ }
+
+ private static String getRunConfigDisplayCmdLine(BatchRunConfiguration runConfig) {
+ final StringBuilder deqpCmdLine = new StringBuilder();
+ if (!runConfig.getGlConfig().isEmpty()) {
+ deqpCmdLine.append("--deqp-gl-config-name=");
+ deqpCmdLine.append(runConfig.getGlConfig());
+ }
+ if (!runConfig.getRotation().isEmpty()) {
+ if (deqpCmdLine.length() != 0) {
+ deqpCmdLine.append(" ");
+ }
+ deqpCmdLine.append("--deqp-screen-rotation=");
+ deqpCmdLine.append(runConfig.getRotation());
+ }
+ if (!runConfig.getSurfaceType().isEmpty()) {
+ if (deqpCmdLine.length() != 0) {
+ deqpCmdLine.append(" ");
+ }
+ deqpCmdLine.append("--deqp-surface-type=");
+ deqpCmdLine.append(runConfig.getSurfaceType());
+ }
+ return deqpCmdLine.toString();
+ }
+
+ /**
+ * Pass given batch tests without running it
+ */
+ private void fakePassTestRunBatch(Collection<TestIdentifier> tests,
+ BatchRunConfiguration runConfig) {
+ for (TestIdentifier test : tests) {
+ CLog.d("Skipping test '%s' invocation in config '%s'", test.toString(),
+ runConfig.getId());
+ mInstanceListerner.skipTest(test);
+ }
+ }
+
+ /**
+ * Pass all remaining tests without running them
+ */
+ private void fakePassTests(ITestInvocationListener listener) {
+ Map <String, String> emptyMap = Collections.emptyMap();
+ for (TestIdentifier test : mRemainingTests) {
+ CLog.d("Skipping test '%s', Opengl ES version not supported", test.toString());
+ listener.testStarted(test);
+ listener.testEnded(test, emptyMap);
+ }
+ mRemainingTests.clear();
}
/**
* Check if device supports OpenGL ES version.
*/
- static boolean isSupportedGles(ITestDevice device, int requiredMajorVersion, int requiredMinorVersion) throws DeviceNotAvailableException {
+ private static boolean isSupportedGles(ITestDevice device, int requiredMajorVersion,
+ int requiredMinorVersion) throws DeviceNotAvailableException {
String roOpenglesVersion = device.getProperty("ro.opengles.version");
if (roOpenglesVersion == null)
@@ -450,6 +941,99 @@
}
/**
+ * Query if rendertarget is supported
+ */
+ private boolean isSupportedGlesRenderConfig(BatchRunConfiguration runConfig)
+ throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ // query if configuration is supported
+ final StringBuilder configCommandLine =
+ new StringBuilder(getRunConfigDisplayCmdLine(runConfig));
+ if (configCommandLine.length() != 0) {
+ configCommandLine.append(" ");
+ }
+ configCommandLine.append("--deqp-gl-major-version=");
+ configCommandLine.append(getGlesMajorVersion());
+ configCommandLine.append(" --deqp-gl-minor-version=");
+ configCommandLine.append(getGlesMinorVersion());
+
+ final String instrumentationName =
+ "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation";
+ final String command = String.format(
+ "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine \"%s\""
+ + " %s",
+ AbiUtils.createAbiFlag(mAbi.getName()), configCommandLine.toString(),
+ instrumentationName);
+
+ final PlatformQueryInstrumentationParser parser = new PlatformQueryInstrumentationParser();
+ mDevice.executeShellCommand(command, parser);
+ parser.flush();
+
+ if (parser.wasSuccessful() && parser.getResultCode() == 0 &&
+ parser.getResultMap().containsKey("Supported")) {
+ if ("Yes".equals(parser.getResultMap().get("Supported"))) {
+ return true;
+ } else if ("No".equals(parser.getResultMap().get("Supported"))) {
+ return false;
+ } else {
+ CLog.e("Capability query did not return a result");
+ throw new CapabilityQueryFailureException();
+ }
+ } else if (parser.wasSuccessful()) {
+ CLog.e("Failed to run capability query. Code: %d, Result: %s",
+ parser.getResultCode(), parser.getResultMap().toString());
+ throw new CapabilityQueryFailureException();
+ } else {
+ CLog.e("Failed to run capability query");
+ throw new CapabilityQueryFailureException();
+ }
+ }
+
+ /**
+ * Return feature set supported by the device
+ */
+ private Set<String> getDeviceFeatures(ITestDevice device)
+ throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ if (mDeviceFeatures == null) {
+ mDeviceFeatures = queryDeviceFeatures(device);
+ }
+ return mDeviceFeatures;
+ }
+
+ /**
+ * Query feature set supported by the device
+ */
+ private static Set<String> queryDeviceFeatures(ITestDevice device)
+ throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ // NOTE: Almost identical code in BaseDevicePolicyTest#hasDeviceFeatures
+ // TODO: Move this logic to ITestDevice.
+ String command = "pm list features";
+ String commandOutput = device.executeShellCommand(command);
+
+ // Extract the id of the new user.
+ HashSet<String> availableFeatures = new HashSet<>();
+ for (String feature: commandOutput.split("\\s+")) {
+ // Each line in the output of the command has the format "feature:{FEATURE_VALUE}".
+ String[] tokens = feature.split(":");
+ if (tokens.length < 2 || !"feature".equals(tokens[0])) {
+ CLog.e("Failed parse features. Unexpect format on line \"%s\"", tokens[0]);
+ throw new CapabilityQueryFailureException();
+ }
+ availableFeatures.add(tokens[1]);
+ }
+ return availableFeatures;
+ }
+
+ private boolean isPortraitClassRotation(String rotation) {
+ return BatchRunConfiguration.ROTATION_PORTRAIT.equals(rotation) ||
+ BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT.equals(rotation);
+ }
+
+ private boolean isLandscapeClassRotation(String rotation) {
+ return BatchRunConfiguration.ROTATION_LANDSCAPE.equals(rotation) ||
+ BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE.equals(rotation);
+ }
+
+ /**
* Install dEQP OnDevice Package
*/
private void installTestApk() throws DeviceNotAvailableException {
@@ -473,60 +1057,53 @@
}
/**
- * {@inheritDoc}
+ * Parse gl nature from package name
*/
- @Override
- public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
- mListener = listener;
-
- if ((mName.equals( "dEQP-GLES3") && isSupportedGles(mDevice, 3, 0))
- || (mName.equals("dEQP-GLES31") && isSupportedGles(mDevice, 3, 1))) {
-
- // Make sure there is no pre-existing package form earlier interrupted test run.
- uninstallTestApk();
- installTestApk();
-
- while (!mTests.isEmpty()) {
- executeTests(listener);
-
- // Set test to failed if it didn't receive test result
- if (mCurrentTestId != null) {
- Map <String, String> emptyMap = Collections.emptyMap();
-
- if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
- ByteArrayInputStreamSource source
- = new ByteArrayInputStreamSource(mCurrentTestLog.getBytes());
-
- mListener.testLog(mCurrentTestId.getClassName() + "."
- + mCurrentTestId.getTestName(), LogDataType.XML, source);
-
- source.cancel();
- }
- if (!mGotTestResult) {
- mListener.testFailed(mCurrentTestId,
- INCOMPLETE_LOG_MESSAGE);
- }
-
- mListener.testEnded(mCurrentTestId, emptyMap);
- mCurrentTestId = null;
- mListener.testRunEnded(0, emptyMap);
- }
- }
-
- uninstallTestApk();
+ private boolean isOpenGlEsPackage() {
+ if ("dEQP-GLES2".equals(mName) || "dEQP-GLES3".equals(mName) ||
+ "dEQP-GLES31".equals(mName)) {
+ return true;
+ } else if ("dEQP-EGL".equals(mName)) {
+ return false;
} else {
- /* Pass all tests if OpenGL ES version is not supported */
- Map <String, String> emptyMap = Collections.emptyMap();
- String id = AbiUtils.createId(mAbi.getName(), mPackageName);
- mListener.testRunStarted(id, mTests.size());
+ throw new IllegalStateException("dEQP runner was created with illegal name");
+ }
+ }
- for (TestIdentifier test : mTests) {
- CLog.d("Skipping test '%s', Opengl ES version not supported", test.toString());
- mListener.testStarted(test);
- mListener.testEnded(test, emptyMap);
- }
+ /**
+ * Check GL support (based on package name)
+ */
+ private boolean isSupportedGles() throws DeviceNotAvailableException {
+ return isSupportedGles(mDevice, getGlesMajorVersion(), getGlesMinorVersion());
+ }
- mListener.testRunEnded(0, emptyMap);
+ /**
+ * Get GL major version (based on package name)
+ */
+ private int getGlesMajorVersion() throws DeviceNotAvailableException {
+ if ("dEQP-GLES2".equals(mName)) {
+ return 2;
+ } else if ("dEQP-GLES3".equals(mName)) {
+ return 3;
+ } else if ("dEQP-GLES31".equals(mName)) {
+ return 3;
+ } else {
+ throw new IllegalStateException("getGlesMajorVersion called for non gles pkg");
+ }
+ }
+
+ /**
+ * Get GL minor version (based on package name)
+ */
+ private int getGlesMinorVersion() throws DeviceNotAvailableException {
+ if ("dEQP-GLES2".equals(mName)) {
+ return 0;
+ } else if ("dEQP-GLES3".equals(mName)) {
+ return 0;
+ } else if ("dEQP-GLES31".equals(mName)) {
+ return 1;
+ } else {
+ throw new IllegalStateException("getGlesMinorVersion called for non gles pkg");
}
}
@@ -534,15 +1111,33 @@
* {@inheritDoc}
*/
@Override
- public void setDevice(ITestDevice device) {
- mDevice = device;
- }
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ final Map<String, String> emptyMap = Collections.emptyMap();
+ final boolean isSupportedApi = !isOpenGlEsPackage() || isSupportedGles();
- /**
- * {@inheritDoc}
- */
- @Override
- public ITestDevice getDevice() {
- return mDevice;
+ listener.testRunStarted(getId(), mRemainingTests.size());
+
+ try {
+ if (isSupportedApi) {
+ // Make sure there is no pre-existing package form earlier interrupted test run.
+ uninstallTestApk();
+ installTestApk();
+
+ mInstanceListerner.setSink(listener);
+ runTests();
+
+ uninstallTestApk();
+ } else {
+ // Pass all tests if OpenGL ES version is not supported
+ fakePassTests(listener);
+ }
+ } catch (CapabilityQueryFailureException ex) {
+ // Platform is not behaving correctly, for example crashing when trying to create
+ // a window. Instead of silenty failing, signal failure by leaving the rest of the
+ // test cases in "NotExecuted" state
+ uninstallTestApk();
+ }
+
+ listener.testRunEnded(0, emptyMap);
}
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
index 372609a..6daebf7 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
@@ -247,7 +247,8 @@
mDigest = generateDigest(testCaseDir, mJarPath);
return vmHostTest;
} else if (DEQP_TEST.equals(mTestType)) {
- DeqpTestRunner deqpTest = new DeqpTestRunner(mAppPackageName, mName, mTests);
+ DeqpTestRunner deqpTest =
+ new DeqpTestRunner(mAppPackageName, mName, mTests, mTestInstanceArguments);
deqpTest.setAbi(mAbi);
return deqpTest;
} else if (NATIVE_TEST.equals(mTestType)) {
diff --git a/tools/tradefed-host/tests/src/com/android/cts/tradefed/testtype/DeqpTestRunnerTest.java b/tools/tradefed-host/tests/src/com/android/cts/tradefed/testtype/DeqpTestRunnerTest.java
index c41793f..2721ec6 100644
--- a/tools/tradefed-host/tests/src/com/android/cts/tradefed/testtype/DeqpTestRunnerTest.java
+++ b/tools/tradefed-host/tests/src/com/android/cts/tradefed/testtype/DeqpTestRunnerTest.java
@@ -21,6 +21,7 @@
import com.android.ddmlib.IShellOutputReceiver;
import com.android.ddmlib.testrunner.ITestRunListener;
import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.testtype.IAbi;
@@ -32,7 +33,10 @@
import java.io.File;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
/**
@@ -45,8 +49,23 @@
private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa";
private static final String INSTRUMENTATION_NAME =
"com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
+ private static final String QUERY_INSTRUMENTATION_NAME =
+ "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation";
private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
+ private static final String ONLY_LANDSCAPE_FEATURES =
+ "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE;
+ private static final String ALL_FEATURES =
+ ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT;
+ private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS;
+
+ static {
+ DEFAULT_INSTANCE_ARGS = new ArrayList<>(1);
+ DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>());
+ DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8");
+ DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified");
+ DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window");
+ }
/**
* {@inheritDoc}
@@ -106,13 +125,15 @@
ITestInvocationListener mockListener
= EasyMock.createStrictMock(ITestInvocationListener.class);
Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
-
tests.add(testId);
+ Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>();
+ instance.put(testId, DEFAULT_INSTANCE_ARGS);
+
DeqpTestRunner deqpTest = new DeqpTestRunner(NAME,
"dEQP-GLES" + Integer.toString(requiredMajorVersion)
+ (requiredMinorVersion > 0 ? Integer.toString(requiredMinorVersion) : ""),
- tests);
+ tests, instance);
deqpTest.setAbi(UnitTests.ABI);
int version = (majorVersion << 16) | minorVersion;
@@ -129,6 +150,8 @@
EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName()))))
.andReturn(null).once();
+ expectRenderConfigQuery(mockDevice, requiredMajorVersion, requiredMinorVersion);
+
EasyMock.expect(mockDevice.executeShellCommand(
EasyMock.eq("rm " + CASE_LIST_FILE_NAME))).andReturn("").once();
@@ -140,7 +163,9 @@
String command = String.format(
"am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \""
- + "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8\" "
+ + "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window\" "
+ "-e deqpLogData \"%s\" %s",
AbiUtils.createAbiFlag(UnitTests.ABI.getName()), LOG_FILE_NAME,
CASE_LIST_FILE_NAME, false, INSTRUMENTATION_NAME);
@@ -188,6 +213,46 @@
EasyMock.verify(mockDevice);
}
+ private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion, int minorVersion)
+ throws Exception {
+ expectRenderConfigQuery(mockDevice, String.format("--deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=%d "
+ + "--deqp-gl-minor-version=%d", majorVersion, minorVersion));
+ }
+
+ private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine)
+ throws Exception {
+ expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes");
+ }
+
+ private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine,
+ String output) throws Exception {
+ final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+ final String command = String.format(
+ "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine "
+ + "\"%s\" %s",
+ AbiUtils.createAbiFlag(UnitTests.ABI.getName()), commandLine, QUERY_INSTRUMENTATION_NAME);
+
+ mockDevice.executeShellCommand(EasyMock.eq(command),
+ EasyMock.<IShellOutputReceiver>notNull());
+
+ EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() {
+ IShellOutputReceiver receiver
+ = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
+
+ receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length());
+ receiver.flush();
+
+ return null;
+ }
+ });
+ }
+
/**
* Test that result code produces correctly pass or fail.
*/
@@ -228,10 +293,12 @@
ITestInvocationListener mockListener
= EasyMock.createStrictMock(ITestInvocationListener.class);
Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
-
tests.add(testId);
- DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests);
+ Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>();
+ instance.put(testId, DEFAULT_INSTANCE_ARGS);
+
+ DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance);
deqpTest.setAbi(UnitTests.ABI);
int version = 3 << 16;
@@ -245,6 +312,8 @@
EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName()))))
.andReturn(null).once();
+ expectRenderConfigQuery(mockDevice, 3, 0);
+
EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
.andReturn("").once();
@@ -256,7 +325,9 @@
String command = String.format(
"am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \""
- + "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8\" "
+ + "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window\" "
+ "-e deqpLogData \"%s\" %s",
AbiUtils.createAbiFlag(UnitTests.ABI.getName()), LOG_FILE_NAME,
CASE_LIST_FILE_NAME, false, INSTRUMENTATION_NAME);
@@ -285,7 +356,8 @@
if (!pass) {
mockListener.testFailed(testId,
- resultCode + ": Detail" + resultCode);
+ "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n"
+ + resultCode + ": Detail" + resultCode);
EasyMock.expectLastCall().once();
}
@@ -412,12 +484,14 @@
ITestInvocationListener mockListener
= EasyMock.createStrictMock(ITestInvocationListener.class);
Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ Map<TestIdentifier, List<Map<String, String>>> instances = new HashMap<>();
for (TestIdentifier id : testIds) {
tests.add(id);
+ instances.put(id, DEFAULT_INSTANCE_ARGS);
}
- DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests);
+ DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instances);
deqpTest.setAbi(UnitTests.ABI);
int version = 3 << 16;
@@ -430,6 +504,8 @@
EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName()))))
.andReturn(null).once();
+ expectRenderConfigQuery(mockDevice, 3, 0);
+
EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
.andReturn("").once();
@@ -441,7 +517,9 @@
String command = String.format(
"am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \""
- + "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8\" "
+ + "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window\" "
+ "-e deqpLogData \"%s\" %s",
AbiUtils.createAbiFlag(UnitTests.ABI.getName()), LOG_FILE_NAME,
CASE_LIST_FILE_NAME, false, INSTRUMENTATION_NAME);
@@ -493,6 +571,269 @@
}
/**
+ * Test that test are left unexecuted if pm list query fails
+ */
+ public void testRun_queryPmListFailure()
+ throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>();
+ instance.put(testId, new ArrayList<Map<String,String>>(1));
+ instance.get(testId).add(new HashMap<String,String>());
+ instance.get(testId).iterator().next().put("glconfig", "rgba8888d24s8");
+ instance.get(testId).iterator().next().put("rotation", "90");
+ instance.get(testId).iterator().next().put("surfacetype", "window");
+
+ DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance);
+ deqpTest.setAbi(UnitTests.ABI);
+ deqpTest.setDevice(mockDevice);
+ deqpTest.setBuildHelper(new StubCtsBuildHelper());
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
+ .andReturn("not a valid format");
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null)
+ .once();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+
+ mockListener.testRunStarted(ID, 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+ deqpTest.run(mockListener);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ /**
+ * Test that test are left unexecuted if renderablity query fails
+ */
+ public void testRun_queryRenderabilityFailure()
+ throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>();
+ instance.put(testId, new ArrayList<Map<String,String>>(1));
+ instance.get(testId).add(new HashMap<String,String>());
+ instance.get(testId).iterator().next().put("glconfig", "rgba8888d24s8");
+ instance.get(testId).iterator().next().put("rotation", "unspecified");
+ instance.get(testId).iterator().next().put("surfacetype", "window");
+
+ DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance);
+ deqpTest.setAbi(UnitTests.ABI);
+ deqpTest.setDevice(mockDevice);
+ deqpTest.setBuildHelper(new StubCtsBuildHelper());
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null)
+ .once();
+
+ expectRenderConfigQueryAndReturn(mockDevice,
+ "--deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", "Maybe?");
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+
+ mockListener.testRunStarted(ID, 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+ deqpTest.run(mockListener);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ /**
+ * Test that orientation is supplied to runner correctly
+ */
+ private void testOrientation(final String rotation, final String featureString)
+ throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
+ final String testPath = "dEQP-GLES3.orientation.test";
+ final String testTrie = "{dEQP-GLES3{orientation{test}}}";
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>();
+ instance.put(testId, new ArrayList<Map<String,String>>(1));
+ instance.get(testId).add(new HashMap<String,String>());
+ instance.get(testId).iterator().next().put("glconfig", "rgba8888d24s8");
+ instance.get(testId).iterator().next().put("rotation", rotation);
+ instance.get(testId).iterator().next().put("surfacetype", "window");
+
+ DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance);
+ deqpTest.setAbi(UnitTests.ABI);
+ deqpTest.setDevice(mockDevice);
+ deqpTest.setBuildHelper(new StubCtsBuildHelper());
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ if (!rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED)) {
+ EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
+ .andReturn(featureString);
+ }
+
+ final boolean isPortraitOrientation =
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_PORTRAIT) ||
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT);
+ final boolean isLandscapeOrientation =
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_LANDSCAPE) ||
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE);
+ final boolean executable =
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED) ||
+ (isPortraitOrientation &&
+ featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) ||
+ (isLandscapeOrientation &&
+ featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE));
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null)
+ .once();
+
+ if (executable) {
+ expectRenderConfigQuery(mockDevice, String.format(
+ "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s "
+ + "--deqp-surface-type=window --deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", rotation));
+
+ EasyMock.expect(
+ mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME))
+ .andReturn(true).once();
+
+ String command = String.format(
+ "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \""
+ + "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=%s "
+ + "--deqp-surface-type=window\" "
+ + "-e deqpLogData \"%s\" %s",
+ AbiUtils.createAbiFlag(UnitTests.ABI.getName()), LOG_FILE_NAME,
+ CASE_LIST_FILE_NAME, rotation, false, INSTRUMENTATION_NAME);
+
+ mockDevice.executeShellCommand(EasyMock.eq(command),
+ EasyMock.<IShellOutputReceiver>notNull());
+
+ EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() {
+ IShellOutputReceiver receiver
+ = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
+
+ receiver.addOutput(output.getBytes(), 0, output.length());
+ receiver.flush();
+
+ return null;
+ }
+ });
+ }
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+
+ mockListener.testRunStarted(ID, 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+ deqpTest.run(mockListener);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ /**
* Test OpeGL ES3 tests on device with OpenGL ES2.
*/
public void testRun_require30DeviceVersion20() throws Exception {
@@ -596,4 +937,681 @@
public void testRun_resultTimeout() throws Exception {
testResultCode("Timeout", false);
}
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationLandscape() throws Exception {
+ testOrientation("90", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationPortrait() throws Exception {
+ testOrientation("0", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationReverseLandscape() throws Exception {
+ testOrientation("270", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationReversePortrait() throws Exception {
+ testOrientation("180", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationUnspecified() throws Exception {
+ testOrientation("unspecified", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation with limited features
+ */
+ public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception {
+ testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation with limited features
+ */
+ public void testRun_orientationLandscapeLimitedFeatures() throws Exception {
+ testOrientation("90", ONLY_LANDSCAPE_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation with limited features
+ */
+ public void testRun_orientationPortraitLimitedFeatures() throws Exception {
+ testOrientation("0", ONLY_LANDSCAPE_FEATURES);
+ }
+
+ /**
+ * Test dEQP unsupported pixel format
+ */
+ public void testRun_unsupportedPixelFormat() throws Exception {
+ final String pixelFormat = "rgba5658d16m4";
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.pixelformat", "test");
+ final String testPath = "dEQP-GLES3.pixelformat.test";
+ final String testTrie = "{dEQP-GLES3{pixelformat{test}}}";
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>();
+ instance.put(testId, new ArrayList<Map<String,String>>(1));
+ instance.get(testId).add(new HashMap<String,String>());
+ instance.get(testId).iterator().next().put("glconfig", pixelFormat);
+ instance.get(testId).iterator().next().put("rotation", "unspecified");
+ instance.get(testId).iterator().next().put("surfacetype", "window");
+
+ DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance);
+ deqpTest.setAbi(UnitTests.ABI);
+ deqpTest.setDevice(mockDevice);
+ deqpTest.setBuildHelper(new StubCtsBuildHelper());
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null)
+ .once();
+
+ expectRenderConfigQueryAndReturn(mockDevice, String.format(
+ "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", pixelFormat), "No");
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+
+ mockListener.testRunStarted(ID, 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+ deqpTest.run(mockListener);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ /**
+ * Test dEQP with multiple instances
+ */
+ public void testRun_multipleInstances() throws Exception {
+ final String instrumentationAnswerConfigA =
+ "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.passall\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.failone\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.crashtwo\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; // early eof
+ final String instrumentationAnswerConfigB =
+ "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.passall\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.crashtwo\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TerminateTestCase-Reason=Magic\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TerminateTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.skipone\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+ final String instrumentationAnswerConfigC =
+ "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.failone\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.crashtwo\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.instances", "passall"),
+ new TestIdentifier("dEQP-GLES3.instances", "failone"),
+ new TestIdentifier("dEQP-GLES3.instances", "crashtwo"),
+ new TestIdentifier("dEQP-GLES3.instances", "skipone"),
+ };
+
+ final String[] testPaths = {
+ "dEQP-GLES3.instances.passall",
+ "dEQP-GLES3.instances.failone",
+ "dEQP-GLES3.instances.crashtwo",
+ "dEQP-GLES3.instances.skipone",
+ };
+
+ Map<String,String> supportedConfigA = new HashMap<>();
+ supportedConfigA.put("glconfig", "rgba8888d24s8");
+ supportedConfigA.put("rotation", "unspecified");
+ supportedConfigA.put("surfacetype", "window");
+
+ Map<String,String> supportedConfigB = new HashMap<>();
+ supportedConfigB.put("glconfig", "rgba8888d24s8");
+ supportedConfigB.put("rotation", "90");
+ supportedConfigB.put("surfacetype", "window");
+
+ Map<String,String> supportedConfigC = new HashMap<>();
+ supportedConfigC.put("glconfig", "rgba8888d24s8");
+ supportedConfigC.put("rotation", "180");
+ supportedConfigC.put("surfacetype", "window");
+
+ Map<String,String> unsupportedConfig = new HashMap<>();
+ unsupportedConfig.put("glconfig", "rgb565d16s0");
+ unsupportedConfig.put("rotation", "unspecified");
+ unsupportedConfig.put("surfacetype", "window");
+
+ Map<TestIdentifier, List<Map<String, String>>> instances = new HashMap<>();
+
+ // pass all
+ instances.put(testIds[0], new ArrayList<Map<String,String>>());
+ instances.get(testIds[0]).add(supportedConfigA);
+ instances.get(testIds[0]).add(supportedConfigB);
+
+ // fail one
+ instances.put(testIds[1], new ArrayList<Map<String,String>>());
+ instances.get(testIds[1]).add(supportedConfigA);
+ instances.get(testIds[1]).add(supportedConfigC);
+
+ // crash two
+ instances.put(testIds[2], new ArrayList<Map<String,String>>());
+ instances.get(testIds[2]).add(supportedConfigA);
+ instances.get(testIds[2]).add(supportedConfigC);
+ instances.get(testIds[2]).add(supportedConfigB);
+
+ // skip one
+ instances.put(testIds[3], new ArrayList<Map<String,String>>());
+ instances.get(testIds[3]).add(supportedConfigB);
+ instances.get(testIds[3]).add(unsupportedConfig);
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ tests.add(id);
+ }
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+
+ DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instances);
+ deqpTest.setAbi(UnitTests.ABI);
+ deqpTest.setDevice(mockDevice);
+ deqpTest.setBuildHelper(new StubCtsBuildHelper());
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+ EasyMock.expect(mockDevice.executeShellCommand("pm list features")).andReturn(ALL_FEATURES)
+ .anyTimes();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null)
+ .once();
+
+ // query config A
+ expectRenderConfigQueryAndReturn(mockDevice, "--deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", "Yes");
+
+ // run config A
+ runInstrumentationLineAndAnswer(mockDevice,
+ "{dEQP-GLES3{instances{passall,failone,crashtwo}}}",
+ "--deqp-caselist-file=" + CASE_LIST_FILE_NAME
+ + " --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window", instrumentationAnswerConfigA);
+
+ // query for config B
+ expectRenderConfigQueryAndReturn(mockDevice, "--deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=90 "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", "Yes");
+
+ // run for config B
+ runInstrumentationLineAndAnswer(mockDevice,
+ "{dEQP-GLES3{instances{passall,crashtwo,skipone}}}",
+ "--deqp-caselist-file=" + CASE_LIST_FILE_NAME
+ + " --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=90 "
+ + "--deqp-surface-type=window", instrumentationAnswerConfigB);
+
+ // query for config C
+ expectRenderConfigQueryAndReturn(mockDevice, "--deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=180 "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", "Yes");
+
+ // run for config C
+ runInstrumentationLineAndAnswer(mockDevice,
+ "{dEQP-GLES3{instances{failone,crashtwo}}}",
+ "--deqp-caselist-file=" + CASE_LIST_FILE_NAME
+ + " --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=180 "
+ + "--deqp-surface-type=window", instrumentationAnswerConfigC);
+
+ // query for unsupported config
+ expectRenderConfigQueryAndReturn(mockDevice, "--deqp-gl-config-name=rgb565d16s0 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", "No");
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+
+ mockListener.testRunStarted(ID, 4);
+ EasyMock.expectLastCall().once();
+
+ // pass all
+ mockListener.testStarted(EasyMock.eq(testIds[0]));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testIds[0]), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ // fail one
+ mockListener.testStarted(EasyMock.eq(testIds[1]));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testFailed(testIds[1],
+ "=== with config {glformat=rgba8888d24s8,rotation=180,surfacetype=window} ===\n"
+ + "Fail: Fail");
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testIds[1]), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ // crash two
+ mockListener.testStarted(EasyMock.eq(testIds[2]));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testFailed(testIds[2],
+ "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n"
+ + "Crash: Incomplete test log\n"
+ + "=== with config {glformat=rgba8888d24s8,rotation=90,surfacetype=window} ===\n"
+ + "Terminated: Magic");
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testIds[2]), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ // skip one
+ mockListener.testStarted(EasyMock.eq(testIds[3]));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testIds[3]), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+ deqpTest.run(mockListener);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ /**
+ * Test dEQP with runner if device is lost during one of multiple instances.
+ */
+ public void testRun_multipleInstancesLossOfDeviceMidInstance() throws Exception {
+ final String instrumentationAnswerFine =
+ "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.loss.instance\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+ final String instrumentationAnswerCrash =
+ "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.loss.instance\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; // early <EOF>
+
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.loss", "instance");
+ final String testPath = "dEQP-GLES3.loss.instance";
+
+ Map<String,String> supportedConfigA = new HashMap<>();
+ supportedConfigA.put("glconfig", "rgba8888d24s8");
+ supportedConfigA.put("rotation", "unspecified");
+ supportedConfigA.put("surfacetype", "window");
+
+ Map<String,String> supportedConfigB = new HashMap<>();
+ supportedConfigB.put("glconfig", "rgba8888d24s8");
+ supportedConfigB.put("rotation", "90");
+ supportedConfigB.put("surfacetype", "window");
+
+ Map<String,String> supportedConfigC = new HashMap<>();
+ supportedConfigC.put("glconfig", "rgba8888d24s8");
+ supportedConfigC.put("rotation", "180");
+ supportedConfigC.put("surfacetype", "window");
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>();
+ instance.put(testId, new ArrayList<Map<String,String>>());
+ instance.get(testId).add(supportedConfigA);
+ instance.get(testId).add(supportedConfigB);
+ instance.get(testId).add(supportedConfigC);
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+
+ DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance);
+ deqpTest.setAbi(UnitTests.ABI);
+ deqpTest.setDevice(mockDevice);
+ deqpTest.setBuildHelper(new StubCtsBuildHelper());
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+ EasyMock.expect(mockDevice.executeShellCommand("pm list features")).andReturn(ALL_FEATURES)
+ .anyTimes();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null)
+ .once();
+
+ // query config A
+ expectRenderConfigQueryAndReturn(mockDevice, "--deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", "Yes");
+
+ // run config A
+ runInstrumentationLineAndAnswer(mockDevice,
+ "{dEQP-GLES3{loss{instance}}}",
+ "--deqp-caselist-file=" + CASE_LIST_FILE_NAME
+ + " --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window", instrumentationAnswerFine);
+
+ // query config B
+ expectRenderConfigQueryAndReturn(mockDevice, "--deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=90 "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", "Yes");
+
+ // run config B
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.pushString("{dEQP-GLES3{loss{instance}}}\n", CASE_LIST_FILE_NAME))
+ .andReturn(true).once();
+
+ String command = String.format(
+ "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \""
+ + "--deqp-caselist-file=%s"
+ + " --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=90 "
+ + "--deqp-surface-type=window\" "
+ + "-e deqpLogData \"%s\" %s",
+ AbiUtils.createAbiFlag(UnitTests.ABI.getName()), LOG_FILE_NAME,
+ CASE_LIST_FILE_NAME, false, INSTRUMENTATION_NAME);
+
+ mockDevice.executeShellCommand(EasyMock.eq(command),
+ EasyMock.<IShellOutputReceiver>notNull());
+
+ EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() throws DeviceNotAvailableException {
+ IShellOutputReceiver receiver
+ = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
+
+ receiver.addOutput(instrumentationAnswerCrash.getBytes(), 0,
+ instrumentationAnswerCrash.length());
+ throw new DeviceNotAvailableException();
+ }
+ });
+
+ mockListener.testRunStarted(ID, 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testFailed(testId,
+ "=== with config {glformat=rgba8888d24s8,rotation=90,surfacetype=window} ===\n"
+ + "Crash: Device lost");
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+
+ try {
+ deqpTest.run(mockListener);
+ fail("did not get DeviceNotAvailableException");
+ } catch (DeviceNotAvailableException ex) {
+ // expected
+ }
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, final String testTrie,
+ final String cmd, final String output) throws Exception {
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME))
+ .andReturn(true).once();
+
+ String command = String.format(
+ "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" "
+ + "-e deqpLogData \"%s\" %s",
+ AbiUtils.createAbiFlag(UnitTests.ABI.getName()), LOG_FILE_NAME,
+ cmd, false, INSTRUMENTATION_NAME);
+
+ mockDevice.executeShellCommand(EasyMock.eq(command),
+ EasyMock.<IShellOutputReceiver>notNull());
+
+ EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() {
+ IShellOutputReceiver receiver
+ = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
+
+ receiver.addOutput(output.getBytes(), 0, output.length());
+ receiver.flush();
+
+ return null;
+ }
+ });
+ }
}